NetCDF  4.7.3
nc4hdf.c
Go to the documentation of this file.
1 /* Copyright 2018, University Corporation for Atmospheric
2  * Research. See the COPYRIGHT file for copying and redistribution
3  * conditions. */
17 #include "config.h"
18 #include "hdf5internal.h"
19 #include <math.h>
20 
21 #ifdef HAVE_INTTYPES_H
22 #define __STDC_FORMAT_MACROS
23 #include <inttypes.h>
24 #endif
25 
26 #define NC_HDF5_MAX_NAME 1024
28 /* WARNING: GLOBAL VARIABLE */
29 
30 /* Define list of registered filters */
31 static NClist* filters = NULL;
32 
41 static int
42 flag_atts_dirty(NCindex *attlist) {
43 
44  NC_ATT_INFO_T *att = NULL;
45  int i;
46 
47  if(attlist == NULL) {
48  return NC_NOERR;
49  }
50 
51  for(i=0;i<ncindexsize(attlist);i++) {
52  att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
53  if(att == NULL) continue;
54  att->dirty = NC_TRUE;
55  }
56 
57  return NC_NOERR;
58 }
59 
76 int
77 rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
78 {
79  NC_VAR_INFO_T *var;
80  NC_GRP_INFO_T *child_grp;
81  int d, i;
82  int retval;
83 
84  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
85  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
86 
87  /* If there are any child groups, attach dimscale there, if needed. */
88  for (i = 0; i < ncindexsize(grp->children); i++)
89  {
90  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
91  assert(child_grp);
92  if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
93  return retval;
94  }
95 
96  /* Find any vars that use this dimension id. */
97  for (i = 0; i < ncindexsize(grp->vars); i++)
98  {
99  NC_HDF5_VAR_INFO_T *hdf5_var;
100 
101  var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
102  assert(var && var->format_var_info);
103  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
104 
105  for (d = 0; d < var->ndims; d++)
106  {
107  if (var->dimids[d] == dimid && !var->dimscale)
108  {
109  LOG((2, "%s: attaching scale for dimid %d to var %s",
110  __func__, var->dimids[d], var->hdr.name));
111  if (var->created)
112  {
113  if (H5DSattach_scale(hdf5_var->hdf_datasetid,
114  dimscaleid, d) < 0)
115  return NC_EHDFERR;
116  var->dimscale_attached[d] = NC_TRUE;
117  }
118  }
119  }
120  }
121  return NC_NOERR;
122 }
123 
140 int
141 rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
142 {
143  NC_VAR_INFO_T *var;
144  NC_GRP_INFO_T *child_grp;
145  int d, i;
146  int retval;
147 
148  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
149  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
150 
151  /* If there are any child groups, detach dimscale there, if needed. */
152  for(i=0;i<ncindexsize(grp->children);i++) {
153  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
154  if(child_grp == NULL) continue;
155  if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
156  return retval;
157  }
158 
159  /* Find any vars that use this dimension id. */
160  for (i = 0; i < ncindexsize(grp->vars); i++)
161  {
162  NC_HDF5_VAR_INFO_T *hdf5_var;
163  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
164  assert(var && var->format_var_info);
165  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
166 
167  for (d = 0; d < var->ndims; d++)
168  {
169  if (var->dimids[d] == dimid && !var->dimscale)
170  {
171  LOG((2, "%s: detaching scale for dimid %d to var %s",
172  __func__, var->dimids[d], var->hdr.name));
173  if (var->created)
174  {
175  if (var->dimscale_attached && var->dimscale_attached[d])
176  {
177  if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
178  dimscaleid, d) < 0)
179  return NC_EHDFERR;
180  var->dimscale_attached[d] = NC_FALSE;
181  }
182  }
183  }
184  }
185  }
186  return NC_NOERR;
187 }
188 
200 int
201 nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
202 {
203  NC_VAR_INFO_T *var;
204  NC_HDF5_VAR_INFO_T *hdf5_var;
205 
206  assert(grp && grp->format_grp_info && dataset);
207 
208  /* Find the requested varid. */
209  if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
210  return NC_ENOTVAR;
211  assert(var && var->hdr.id == varid && var->format_var_info);
212  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
213 
214  /* Open this dataset if necessary. */
215  if (!hdf5_var->hdf_datasetid)
216  {
217  NC_HDF5_GRP_INFO_T *hdf5_grp;
218  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
219 
220  if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
221  var->hdr.name, H5P_DEFAULT)) < 0)
222  return NC_ENOTVAR;
223  }
224 
225  *dataset = hdf5_var->hdf_datasetid;
226 
227  return NC_NOERR;
228 }
229 
241 int
242 nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
243 {
244  size_t size;
245  int retval;
246 
247  /* Find out how much space we need for this type's fill value. */
248  if (var->type_info->nc_type_class == NC_VLEN)
249  size = sizeof(nc_vlen_t);
250  else if (var->type_info->nc_type_class == NC_STRING)
251  size = sizeof(char *);
252  else
253  {
254  if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size)))
255  return retval;
256  }
257  assert(size);
258 
259  /* Allocate the space. */
260  if (!((*fillp) = calloc(1, size)))
261  return NC_ENOMEM;
262 
263  /* If the user has set a fill_value for this var, use, otherwise
264  * find the default fill value. */
265  if (var->fill_value)
266  {
267  LOG((4, "Found a fill value for var %s", var->hdr.name));
268  if (var->type_info->nc_type_class == NC_VLEN)
269  {
270  nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
271  size_t basetypesize = 0;
272 
273  if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
274  return retval;
275 
276  fv_vlen->len = in_vlen->len;
277  if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
278  {
279  free(*fillp);
280  *fillp = NULL;
281  return NC_ENOMEM;
282  }
283  memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
284  }
285  else if (var->type_info->nc_type_class == NC_STRING)
286  {
287  if (*(char **)var->fill_value)
288  if (!(**(char ***)fillp = strdup(*(char **)var->fill_value)))
289  {
290  free(*fillp);
291  *fillp = NULL;
292  return NC_ENOMEM;
293  }
294  }
295  else
296  memcpy((*fillp), var->fill_value, size);
297  }
298  else
299  {
300  if (nc4_get_default_fill_value(var->type_info, *fillp))
301  {
302  /* Note: release memory, but don't return error on failure */
303  free(*fillp);
304  *fillp = NULL;
305  }
306  }
307 
308  return NC_NOERR;
309 }
310 
328 int
329 nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
330  hid_t *hdf_typeid, int endianness)
331 {
332  NC_TYPE_INFO_T *type;
333  hid_t typeid = 0;
334  int retval = NC_NOERR;
335 
336  assert(hdf_typeid && h5);
337 
338  *hdf_typeid = -1;
339 
340  /* Determine an appropriate HDF5 datatype */
341  if (xtype == NC_NAT)
342  return NC_EBADTYPE;
343  else if (xtype == NC_CHAR || xtype == NC_STRING)
344  {
345  /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
346  if (xtype == NC_CHAR)
347  {
348  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
349  return NC_EHDFERR;
350  if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
351  BAIL(NC_EVARMETA);
352  if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
353  BAIL(NC_EVARMETA);
354 
355  /* Take ownership of the newly created HDF5 datatype */
356  *hdf_typeid = typeid;
357  typeid = 0;
358  }
359  else
360  {
361  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
362  return NC_EHDFERR;
363  if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
364  BAIL(NC_EVARMETA);
365  if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
366  BAIL(NC_EVARMETA);
367 
368  /* Take ownership of the newly created HDF5 datatype */
369  *hdf_typeid = typeid;
370  typeid = 0;
371  }
372  }
373  else
374  {
375  /* All other types use an existing HDF5 datatype */
376  switch (xtype)
377  {
378  case NC_BYTE: /* signed 1 byte integer */
379  if (endianness == NC_ENDIAN_LITTLE)
380  typeid = H5T_STD_I8LE;
381  else if (endianness == NC_ENDIAN_BIG)
382  typeid = H5T_STD_I8BE;
383  else
384  typeid = H5T_NATIVE_SCHAR;
385  break;
386 
387  case NC_SHORT: /* signed 2 byte integer */
388  if (endianness == NC_ENDIAN_LITTLE)
389  typeid = H5T_STD_I16LE;
390  else if (endianness == NC_ENDIAN_BIG)
391  typeid = H5T_STD_I16BE;
392  else
393  typeid = H5T_NATIVE_SHORT;
394  break;
395 
396  case NC_INT:
397  if (endianness == NC_ENDIAN_LITTLE)
398  typeid = H5T_STD_I32LE;
399  else if (endianness == NC_ENDIAN_BIG)
400  typeid = H5T_STD_I32BE;
401  else
402  typeid = H5T_NATIVE_INT;
403  break;
404 
405  case NC_UBYTE:
406  if (endianness == NC_ENDIAN_LITTLE)
407  typeid = H5T_STD_U8LE;
408  else if (endianness == NC_ENDIAN_BIG)
409  typeid = H5T_STD_U8BE;
410  else
411  typeid = H5T_NATIVE_UCHAR;
412  break;
413 
414  case NC_USHORT:
415  if (endianness == NC_ENDIAN_LITTLE)
416  typeid = H5T_STD_U16LE;
417  else if (endianness == NC_ENDIAN_BIG)
418  typeid = H5T_STD_U16BE;
419  else
420  typeid = H5T_NATIVE_USHORT;
421  break;
422 
423  case NC_UINT:
424  if (endianness == NC_ENDIAN_LITTLE)
425  typeid = H5T_STD_U32LE;
426  else if (endianness == NC_ENDIAN_BIG)
427  typeid = H5T_STD_U32BE;
428  else
429  typeid = H5T_NATIVE_UINT;
430  break;
431 
432  case NC_INT64:
433  if (endianness == NC_ENDIAN_LITTLE)
434  typeid = H5T_STD_I64LE;
435  else if (endianness == NC_ENDIAN_BIG)
436  typeid = H5T_STD_I64BE;
437  else
438  typeid = H5T_NATIVE_LLONG;
439  break;
440 
441  case NC_UINT64:
442  if (endianness == NC_ENDIAN_LITTLE)
443  typeid = H5T_STD_U64LE;
444  else if (endianness == NC_ENDIAN_BIG)
445  typeid = H5T_STD_U64BE;
446  else
447  typeid = H5T_NATIVE_ULLONG;
448  break;
449 
450  case NC_FLOAT:
451  if (endianness == NC_ENDIAN_LITTLE)
452  typeid = H5T_IEEE_F32LE;
453  else if (endianness == NC_ENDIAN_BIG)
454  typeid = H5T_IEEE_F32BE;
455  else
456  typeid = H5T_NATIVE_FLOAT;
457  break;
458 
459  case NC_DOUBLE:
460  if (endianness == NC_ENDIAN_LITTLE)
461  typeid = H5T_IEEE_F64LE;
462  else if (endianness == NC_ENDIAN_BIG)
463  typeid = H5T_IEEE_F64BE;
464  else
465  typeid = H5T_NATIVE_DOUBLE;
466  break;
467 
468  default:
469  /* Maybe this is a user defined type? */
470  if (nc4_find_type(h5, xtype, &type))
471  return NC_EBADTYPE;
472  if (!type)
473  return NC_EBADTYPE;
474  typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
475  break;
476  }
477  assert(typeid);
478 
479  /* Copy the HDF5 datatype, so the function operates uniformly */
480  if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
481  return NC_EHDFERR;
482  typeid = 0;
483  }
484  assert(*hdf_typeid != -1);
485 
486 exit:
487  if (typeid > 0 && H5Tclose(typeid) < 0)
488  BAIL2(NC_EHDFERR);
489  return retval;
490 }
491 
506 static int
507 put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
508 {
509  NC_HDF5_GRP_INFO_T *hdf5_grp;
510  hid_t datasetid = 0, locid;
511  hid_t attid = 0, spaceid = 0, file_typeid = 0;
512  hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
513  hsize_t dims[1]; /* netcdf attributes always 1-D. */
514  htri_t attr_exists;
515  void *data;
516  int phoney_data = 99;
517  int retval = NC_NOERR;
518 
519  assert(att->hdr.name && grp && grp->format_grp_info);
520  LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
521  "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
522  att->nc_typeid, att->len));
523 
524  /* Get HDF5-specific group info. */
525  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
526 
527  /* If the file is read-only, return an error. */
528  if (grp->nc4_info->no_write)
529  BAIL(NC_EPERM);
530 
531  /* Get the hid to attach the attribute to, or read it from. */
532  if (varid == NC_GLOBAL)
533  locid = hdf5_grp->hdf_grpid;
534  else
535  {
536  if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
537  BAIL(retval);
538  locid = datasetid;
539  }
540 
541  /* Get the length ready, and find the HDF type we'll be
542  * writing. */
543  dims[0] = att->len;
544  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
545  &file_typeid, 0)))
546  BAIL(retval);
547 
548  /* Even if the length is zero, HDF5 won't let me write with a
549  * NULL pointer. So if the length of the att is zero, point to
550  * some phoney data (which won't be written anyway.)*/
551  if (!dims[0])
552  data = &phoney_data;
553  else if (att->data)
554  data = att->data;
555  else if (att->stdata)
556  data = att->stdata;
557  else
558  data = att->vldata;
559 
560  /* NC_CHAR types require some extra work. The space ID is set to
561  * scalar, and the type is told how long the string is. If it's
562  * really zero length, set the size to 1. (The fact that it's
563  * really zero will be marked by the NULL dataspace, but HDF5
564  * doesn't allow me to set the size of the type to zero.)*/
565  if (att->nc_typeid == NC_CHAR)
566  {
567  size_t string_size = dims[0];
568  if (!string_size)
569  {
570  string_size = 1;
571  if ((spaceid = H5Screate(H5S_NULL)) < 0)
572  BAIL(NC_EATTMETA);
573  }
574  else
575  {
576  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
577  BAIL(NC_EATTMETA);
578  }
579  if (H5Tset_size(file_typeid, string_size) < 0)
580  BAIL(NC_EATTMETA);
581  if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
582  BAIL(NC_EATTMETA);
583  }
584  else
585  {
586  if (!att->len)
587  {
588  if ((spaceid = H5Screate(H5S_NULL)) < 0)
589  BAIL(NC_EATTMETA);
590  }
591  else
592  {
593  if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
594  BAIL(NC_EATTMETA);
595  }
596  }
597 
598  /* Does the att exists already? */
599  if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
600  BAIL(NC_EHDFERR);
601  if (attr_exists)
602  {
603  hssize_t npoints;
604 
605  /* Open the attribute. */
606  if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
607  BAIL(NC_EATTMETA);
608 
609  /* Find the type of the existing attribute. */
610  if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
611  BAIL(NC_EATTMETA);
612 
613  /* How big is the attribute? */
614  if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
615  BAIL(NC_EATTMETA);
616  if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
617  BAIL(NC_EATTMETA);
618 
619  /* For text attributes the size is specified in the datatype
620  and it is enough to compare types using H5Tequal(). */
621  if (!H5Tequal(file_typeid, existing_att_typeid) ||
622  (att->nc_typeid != NC_CHAR && npoints != att->len))
623  {
624  /* The attribute exists but we cannot re-use it. */
625 
626  /* Delete the attribute. */
627  if (H5Adelete(locid, att->hdr.name) < 0)
628  BAIL(NC_EHDFERR);
629 
630  /* Re-create the attribute with the type and length
631  reflecting the new value (or values). */
632  if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid,
633  H5P_DEFAULT)) < 0)
634  BAIL(NC_EATTMETA);
635 
636  /* Write the values, (even if length is zero). */
637  if (H5Awrite(attid, file_typeid, data) < 0)
638  BAIL(NC_EATTMETA);
639  }
640  else
641  {
642  /* The attribute exists and we can re-use it. */
643 
644  /* Write the values, re-using the existing attribute. */
645  if (H5Awrite(existing_attid, file_typeid, data) < 0)
646  BAIL(NC_EATTMETA);
647  }
648  }
649  else
650  {
651  /* The attribute does not exist yet. */
652 
653  /* Create the attribute. */
654  if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid,
655  H5P_DEFAULT)) < 0)
656  BAIL(NC_EATTMETA);
657 
658  /* Write the values, (even if length is zero). */
659  if (H5Awrite(attid, file_typeid, data) < 0)
660  BAIL(NC_EATTMETA);
661  }
662 
663 exit:
664  if (file_typeid && H5Tclose(file_typeid))
665  BAIL2(NC_EHDFERR);
666  if (attid > 0 && H5Aclose(attid) < 0)
667  BAIL2(NC_EHDFERR);
668  if (existing_att_typeid && H5Tclose(existing_att_typeid))
669  BAIL2(NC_EHDFERR);
670  if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
671  BAIL2(NC_EHDFERR);
672  if (spaceid > 0 && H5Sclose(spaceid) < 0)
673  BAIL2(NC_EHDFERR);
674  if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
675  BAIL2(NC_EHDFERR);
676  return retval;
677 }
678 
690 static int
691 write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
692 {
693  NC_ATT_INFO_T *att;
694  int retval;
695  int i;
696 
697  for(i = 0; i < ncindexsize(attlist); i++)
698  {
699  att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
700  assert(att);
701  if (att->dirty)
702  {
703  LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
704  if ((retval = put_att_grpa(grp, varid, att)))
705  return retval;
706  att->dirty = NC_FALSE;
707  att->created = NC_TRUE;
708  }
709  }
710  return NC_NOERR;
711 }
712 
726 static int
727 write_coord_dimids(NC_VAR_INFO_T *var)
728 {
729  NC_HDF5_VAR_INFO_T *hdf5_var;
730  hsize_t coords_len[1];
731  hid_t c_spaceid = -1, c_attid = -1;
732  int retval = NC_NOERR;
733 
734  assert(var && var->format_var_info);
735 
736  /* Get HDF5-specific var info. */
737  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
738 
739  /* Set up space for attribute. */
740  coords_len[0] = var->ndims;
741  if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
742  BAIL(NC_EHDFERR);
743 
744  /* Create the attribute. */
745  if ((c_attid = H5Acreate(hdf5_var->hdf_datasetid, COORDINATES,
746  H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
747  BAIL(NC_EHDFERR);
748 
749  /* Write our attribute. */
750  if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
751  BAIL(NC_EHDFERR);
752 
753 exit:
754  if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
755  BAIL2(NC_EHDFERR);
756  if (c_attid >= 0 && H5Aclose(c_attid) < 0)
757  BAIL2(NC_EHDFERR);
758  return retval;
759 }
760 
771 static int
772 write_netcdf4_dimid(hid_t datasetid, int dimid)
773 {
774  hid_t dimid_spaceid = -1, dimid_attid = -1;
775  htri_t attr_exists;
776  int retval = NC_NOERR;
777 
778  /* Create the space. */
779  if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
780  BAIL(NC_EHDFERR);
781 
782  /* Does the attribute already exist? If so, don't try to create it. */
783  if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
784  BAIL(NC_EHDFERR);
785  if (attr_exists)
786  dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
787  H5P_DEFAULT, H5P_DEFAULT);
788  else
789  /* Create the attribute if needed. */
790  dimid_attid = H5Acreate(datasetid, NC_DIMID_ATT_NAME,
791  H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
792  if (dimid_attid < 0)
793  BAIL(NC_EHDFERR);
794 
795 
796  /* Write it. */
797  LOG((4, "%s: writing secret dimid %d", __func__, dimid));
798  if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
799  BAIL(NC_EHDFERR);
800 
801 exit:
802  /* Close stuff*/
803  if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
804  BAIL2(NC_EHDFERR);
805  if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
806  BAIL2(NC_EHDFERR);
807 
808  return retval;
809 }
810 
825 static int
826 var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
827 {
828  NC_HDF5_GRP_INFO_T *hdf5_grp;
829  NC_HDF5_VAR_INFO_T *hdf5_var;
830  hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
831  hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
832  int d;
833  void *fillp = NULL;
834  NC_DIM_INFO_T *dim = NULL;
835  char *name_to_use;
836  int retval;
837 
838  assert(grp && grp->format_grp_info && var && var->format_var_info);
839 
840  LOG((3, "%s:: name %s", __func__, var->hdr.name));
841 
842  /* Get HDF5-specific group and var info. */
843  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
844  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
845 
846  /* Scalar or not, we need a creation property list. */
847  if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
848  BAIL(NC_EHDFERR);
849  if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
850  BAIL(NC_EHDFERR);
851 
852  /* Turn off object tracking times in HDF5. */
853  if (H5Pset_obj_track_times(plistid, 0) < 0)
854  BAIL(NC_EHDFERR);
855 
856  /* Find the HDF5 type of the dataset. */
857  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
858  var->type_info->endianness)))
859  BAIL(retval);
860 
861  /* Figure out what fill value to set, if any. */
862  if (var->no_fill)
863  {
864  /* Required to truly turn HDF5 fill values off */
865  if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
866  BAIL(NC_EHDFERR);
867  }
868  else
869  {
870  if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
871  BAIL(retval);
872 
873  /* If there is a fill value, set it. */
874  if (fillp)
875  {
876  if (var->type_info->nc_type_class == NC_STRING)
877  {
878  if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
879  BAIL(NC_EHDFERR);
880  }
881  else
882  {
883  /* The fill value set in HDF5 must always be presented as
884  * a native type, even if the endianness for this dataset
885  * is non-native. HDF5 will translate the fill value to
886  * the target endiannesss. */
887  hid_t fill_typeid = 0;
888 
889  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
891  BAIL(retval);
892  if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
893  {
894  if (H5Tclose(fill_typeid) < 0)
895  BAIL(NC_EHDFERR);
896  BAIL(NC_EHDFERR);
897  }
898  if (H5Tclose(fill_typeid) < 0)
899  BAIL(NC_EHDFERR);
900  }
901  }
902  }
903 
904  /* If the user wants to shuffle the data, set that up now. */
905  if (var->shuffle) {
906  if (H5Pset_shuffle(plistid) < 0)
907  BAIL(NC_EHDFERR);
908  }
909 
910  /* If the user wants to deflate the data, set that up now. */
911  if (var->deflate) {
912  if (H5Pset_deflate(plistid, var->deflate_level) < 0)
913  BAIL(NC_EHDFERR);
914  } else if(var->filterid) {
915  /* Handle szip case here */
916  if(var->filterid == H5Z_FILTER_SZIP) {
917  int options_mask;
918  int bits_per_pixel;
919  if(var->nparams != 2)
920  BAIL(NC_EFILTER);
921  options_mask = (int)var->params[0];
922  bits_per_pixel = (int)var->params[1];
923  if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
924  BAIL(NC_EFILTER);
925  } else {
926  herr_t code = H5Pset_filter(plistid, var->filterid, H5Z_FLAG_MANDATORY, var->nparams, var->params);
927  if(code < 0) {
928  BAIL(NC_EFILTER);
929  }
930  }
931  }
932 
933  /* If the user wants to fletcher error correction, set that up now. */
934  if (var->fletcher32)
935  if (H5Pset_fletcher32(plistid) < 0)
936  BAIL(NC_EHDFERR);
937 
938  /* If ndims non-zero, get info for all dimensions. We look up the
939  dimids and get the len of each dimension. We need this to create
940  the space for the dataset. In netCDF a dimension length of zero
941  means an unlimited dimension. */
942  if (var->ndims)
943  {
944  int unlimdim = 0;
945 
946  /* Check to see if any unlimited dimensions are used in this var. */
947  for (d = 0; d < var->ndims; d++) {
948  dim = var->dim[d];
949  assert(dim && dim->hdr.id == var->dimids[d]);
950  if (dim->unlimited)
951  unlimdim++;
952  }
953 
954  /* If there are no unlimited dims, and no filters, and the user
955  * has not specified chunksizes, use contiguous variable for
956  * better performance. */
957  if (!var->shuffle && !var->deflate && !var->fletcher32 &&
958  (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
959  var->contiguous = NC_TRUE;
960 
961  /* Gather current & maximum dimension sizes, along with chunk sizes */
962  for (d = 0; d < var->ndims; d++)
963  {
964  dim = var->dim[d];
965  assert(dim && dim->hdr.id == var->dimids[d]);
966  dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
967  maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
968  if (!var->contiguous) {
969  if (var->chunksizes[d])
970  chunksize[d] = var->chunksizes[d];
971  else
972  {
973  size_t type_size;
974  if (var->type_info->nc_type_class == NC_STRING)
975  type_size = sizeof(char *);
976  else
977  type_size = var->type_info->size;
978 
979  /* Unlimited dim always gets chunksize of 1. */
980  if (dim->unlimited)
981  chunksize[d] = 1;
982  else
983  chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
984  1/(double)(var->ndims - unlimdim));
985 
986  /* If the chunksize is greater than the dim
987  * length, make it the dim length. */
988  if (!dim->unlimited && chunksize[d] > dim->len)
989  chunksize[d] = dim->len;
990 
991  /* Remember the computed chunksize */
992  var->chunksizes[d] = chunksize[d];
993  }
994  }
995  }
996 
997  if (var->contiguous)
998  {
999  if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
1000  BAIL(NC_EHDFERR);
1001  }
1002  else
1003  {
1004  if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
1005  BAIL(NC_EHDFERR);
1006  }
1007 
1008  /* Create the dataspace. */
1009  if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
1010  BAIL(NC_EHDFERR);
1011  }
1012  else
1013  {
1014  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1015  BAIL(NC_EHDFERR);
1016  }
1017 
1018  /* Turn on creation order tracking. */
1019  if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
1020  H5P_CRT_ORDER_INDEXED) < 0)
1021  BAIL(NC_EHDFERR);
1022 
1023  /* Set per-var chunk cache, for chunked datasets. */
1024  if (!var->contiguous && var->chunk_cache_size)
1025  if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
1026  var->chunk_cache_size, var->chunk_cache_preemption) < 0)
1027  BAIL(NC_EHDFERR);
1028 
1029  /* At long last, create the dataset. */
1030  name_to_use = var->hdf5_name ? var->hdf5_name : var->hdr.name;
1031  LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
1032  name_to_use, typeid));
1033  if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
1034  spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
1035  BAIL(NC_EHDFERR);
1036  var->created = NC_TRUE;
1037  var->is_new_var = NC_FALSE;
1038 
1039  /* Always write the hidden coordinates attribute, which lists the
1040  * dimids of this var. When present, this speeds opens. When no
1041  * present, dimscale matching is used. */
1042  if (var->ndims > 1)
1043  if ((retval = write_coord_dimids(var)))
1044  BAIL(retval);
1045 
1046  /* If this is a dimscale, mark it as such in the HDF5 file. Also
1047  * find the dimension info and store the dataset id of the dimscale
1048  * dataset. */
1049  if (var->dimscale)
1050  {
1051  if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1052  BAIL(NC_EHDFERR);
1053 
1054  /* If this is a multidimensional coordinate variable, write a
1055  * coordinates attribute. */
1056  /* if (var->ndims > 1) */
1057  /* if ((retval = write_coord_dimids(var))) */
1058  /* BAIL(retval); */
1059 
1060  /* If desired, write the netCDF dimid. */
1061  if (write_dimid)
1062  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1063  BAIL(retval);
1064  }
1065 
1066  /* Write attributes for this var. */
1067  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1068  BAIL(retval);
1069  var->attr_dirty = NC_FALSE;
1070 
1071 exit:
1072  if (typeid > 0 && H5Tclose(typeid) < 0)
1073  BAIL2(NC_EHDFERR);
1074  if (plistid > 0 && H5Pclose(plistid) < 0)
1075  BAIL2(NC_EHDFERR);
1076  if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1077  BAIL2(NC_EHDFERR);
1078  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1079  BAIL2(NC_EHDFERR);
1080  if (fillp)
1081  {
1082  if (var->type_info->nc_type_class == NC_VLEN)
1083  nc_free_vlen((nc_vlen_t *)fillp);
1084  else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1085  free(*(char **)fillp);
1086  free(fillp);
1087  }
1088 
1089  return retval;
1090 }
1091 
1102 int
1103 nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1104 {
1105  size_t chunk_size_bytes = 1;
1106  int d;
1107  int retval;
1108 
1109  /* Nothing to be done. */
1110  if (var->contiguous)
1111  return NC_NOERR;
1112 #ifdef USE_PARALLEL4
1113  return NC_NOERR;
1114 #endif
1115 
1116  /* How many bytes in the chunk? */
1117  for (d = 0; d < var->ndims; d++)
1118  chunk_size_bytes *= var->chunksizes[d];
1119  if (var->type_info->size)
1120  chunk_size_bytes *= var->type_info->size;
1121  else
1122  chunk_size_bytes *= sizeof(char *);
1123 
1124  /* If the chunk cache is too small, and the user has not changed
1125  * the default value of the chunk cache size, then increase the
1126  * size of the cache. */
1127  if (var->chunk_cache_size == CHUNK_CACHE_SIZE)
1128  if (chunk_size_bytes > var->chunk_cache_size)
1129  {
1130  var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1131  if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE)
1132  var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE;
1133  if ((retval = nc4_reopen_dataset(grp, var)))
1134  return retval;
1135  }
1136 
1137  return NC_NOERR;
1138 }
1139 
1155 static int
1156 commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1157 {
1158  NC_HDF5_GRP_INFO_T *hdf5_grp;
1159  NC_HDF5_TYPE_INFO_T *hdf5_type;
1160  hid_t base_hdf_typeid;
1161  int retval;
1162 
1163  assert(grp && grp->format_grp_info && type && type->format_type_info);
1164 
1165  /* Get HDF5-specific group and type info. */
1166  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1167  hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1168 
1169  /* Did we already record this type? */
1170  if (type->committed)
1171  return NC_NOERR;
1172 
1173  /* Is this a compound type? */
1174  if (type->nc_type_class == NC_COMPOUND)
1175  {
1176  NC_FIELD_INFO_T *field;
1177  hid_t hdf_base_typeid, hdf_typeid;
1178  int i;
1179 
1180  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1181  return NC_EHDFERR;
1182  LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1183  hdf5_type->hdf_typeid));
1184 
1185  for(i=0;i<nclistlength(type->u.c.field);i++)
1186  {
1187  field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1188  assert(field);
1189  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1190  &hdf_base_typeid, type->endianness)))
1191  return retval;
1192 
1193  /* If this is an array, create a special array type. */
1194  if (field->ndims)
1195  {
1196  int d;
1197  hsize_t dims[NC_MAX_VAR_DIMS];
1198 
1199  for (d = 0; d < field->ndims; d++)
1200  dims[d] = field->dim_size[d];
1201  if ((hdf_typeid = H5Tarray_create(hdf_base_typeid, field->ndims,
1202  dims, NULL)) < 0)
1203  {
1204  if (H5Tclose(hdf_base_typeid) < 0)
1205  return NC_EHDFERR;
1206  return NC_EHDFERR;
1207  }
1208  if (H5Tclose(hdf_base_typeid) < 0)
1209  return NC_EHDFERR;
1210  }
1211  else
1212  hdf_typeid = hdf_base_typeid;
1213  LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1214  field->offset, hdf_typeid));
1215  if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1216  hdf_typeid) < 0)
1217  return NC_EHDFERR;
1218  if (H5Tclose(hdf_typeid) < 0)
1219  return NC_EHDFERR;
1220  }
1221  }
1222  else if (type->nc_type_class == NC_VLEN)
1223  {
1224  /* Find the HDF typeid of the base type of this vlen. */
1225  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1226  &base_hdf_typeid, type->endianness)))
1227  return retval;
1228 
1229  /* Create a vlen type. */
1230  if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1231  return NC_EHDFERR;
1232  }
1233  else if (type->nc_type_class == NC_OPAQUE)
1234  {
1235  /* Create the opaque type. */
1236  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1237  return NC_EHDFERR;
1238  }
1239  else if (type->nc_type_class == NC_ENUM)
1240  {
1241  NC_ENUM_MEMBER_INFO_T *enum_m;
1242  int i;
1243 
1244  if (nclistlength(type->u.e.enum_member) == 0)
1245  return NC_EINVAL;
1246 
1247  /* Find the HDF typeid of the base type of this enum. */
1248  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1249  &base_hdf_typeid, type->endianness)))
1250  return retval;
1251 
1252  /* Create an enum type. */
1253  if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1254  return NC_EHDFERR;
1255 
1256  /* Add all the members to the HDF5 type. */
1257  for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1258  enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1259  if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1260  return NC_EHDFERR;
1261  }
1262  }
1263  else
1264  {
1265  LOG((0, "Unknown class: %d", type->nc_type_class));
1266  return NC_EBADTYPE;
1267  }
1268 
1269  /* Commit the type. */
1270  if (H5Tcommit(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1271  return NC_EHDFERR;
1272  type->committed = NC_TRUE;
1273  LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1274  hdf5_type->hdf_typeid));
1275 
1276  /* Later we will always use the native typeid. In this case, it is
1277  * a copy of the same type pointed to by hdf_typeid, but it's
1278  * easier to maintain a copy. */
1279  if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1280  H5T_DIR_DEFAULT)) < 0)
1281  return NC_EHDFERR;
1282 
1283  return NC_NOERR;
1284 }
1285 
1296 static int
1297 write_nc3_strict_att(hid_t hdf_grpid)
1298 {
1299  hid_t attid = 0, spaceid = 0;
1300  int one = 1;
1301  int retval = NC_NOERR;
1302  htri_t attr_exists;
1303 
1304  /* If the attribute already exists, call that a success and return
1305  * NC_NOERR. */
1306  if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1307  return NC_EHDFERR;
1308  if (attr_exists)
1309  return NC_NOERR;
1310 
1311  /* Create the attribute to mark this as a file that needs to obey
1312  * strict netcdf-3 rules. */
1313  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1314  BAIL(NC_EFILEMETA);
1315  if ((attid = H5Acreate(hdf_grpid, NC3_STRICT_ATT_NAME,
1316  H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1317  BAIL(NC_EFILEMETA);
1318  if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1319  BAIL(NC_EFILEMETA);
1320 
1321 exit:
1322  if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1323  BAIL2(NC_EFILEMETA);
1324  if (attid > 0 && (H5Aclose(attid) < 0))
1325  BAIL2(NC_EFILEMETA);
1326  return retval;
1327 }
1328 
1341 static int
1342 create_group(NC_GRP_INFO_T *grp)
1343 {
1344  NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1345  hid_t gcpl_id = -1;
1346  int retval = NC_NOERR;;
1347 
1348  assert(grp && grp->format_grp_info && grp->parent &&
1349  grp->parent->format_grp_info);
1350 
1351  /* Get HDF5 specific group info for group and parent. */
1352  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1353  parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1354  assert(parent_hdf5_grp->hdf_grpid);
1355 
1356  /* Create group, with link_creation_order set in the group
1357  * creation property list. */
1358  if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1359  BAIL(NC_EHDFERR);
1360 
1361  /* Set track_times to be FALSE. */
1362  if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1363  BAIL(NC_EHDFERR);
1364 
1365  /* Tell HDF5 to keep track of objects in creation order. */
1366  if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1367  BAIL(NC_EHDFERR);
1368 
1369  /* Tell HDF5 to keep track of attributes in creation order. */
1370  if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1371  BAIL(NC_EHDFERR);
1372 
1373  /* Create the group. */
1374  if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1375  H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1376  BAIL(NC_EHDFERR);
1377 
1378 exit:
1379  if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1380  BAIL2(NC_EHDFERR);
1381  if (retval)
1382  if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1383  BAIL2(NC_EHDFERR);
1384  return retval;
1385 }
1386 
1398 static int
1399 attach_dimscales(NC_GRP_INFO_T *grp)
1400 {
1401  NC_VAR_INFO_T *var;
1402  NC_HDF5_VAR_INFO_T *hdf5_var;
1403  int d, v;
1404 
1405  /* Attach dimension scales. */
1406  for (v = 0; v < ncindexsize(grp->vars); v++)
1407  {
1408  /* Get pointer to var and HDF5-specific var info. */
1409  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1410  assert(var && var->format_var_info);
1411  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1412 
1413  /* Scales themselves do not attach. But I really wish they
1414  * would. */
1415  if (var->dimscale)
1416  continue;
1417 
1418  /* Find the scale for each dimension, if any, and attach it. */
1419  for (d = 0; d < var->ndims; d++)
1420  {
1421  /* Is there a dimscale for this dimension? */
1422  if (var->dimscale_attached)
1423  {
1424  if (!var->dimscale_attached[d])
1425  {
1426  hid_t dsid; /* Dataset ID for dimension */
1427  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1428  var->dim[d]->format_dim_info);
1429 
1430  LOG((2, "%s: attaching scale for dimid %d to var %s",
1431  __func__, var->dimids[d], var->hdr.name));
1432 
1433  /* Find dataset ID for dimension */
1434  if (var->dim[d]->coord_var)
1435  dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1436  else
1437  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1438  assert(dsid > 0);
1439 
1440  /* Attach the scale. */
1441  if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1442  return NC_EHDFERR;
1443  var->dimscale_attached[d] = NC_TRUE;
1444  }
1445  }
1446  }
1447  }
1448 
1449  return NC_NOERR;
1450 }
1451 
1462 static int
1463 var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1464 {
1465  htri_t link_exists;
1466 
1467  /* Reset the boolean */
1468  *exists = NC_FALSE;
1469 
1470  /* Check if the object name exists in the group */
1471  if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1472  return NC_EHDFERR;
1473  if (link_exists)
1474  {
1475  H5G_stat_t statbuf;
1476 
1477  /* Get info about the object */
1478  if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1479  return NC_EHDFERR;
1480 
1481  if (H5G_DATASET == statbuf.type)
1482  *exists = NC_TRUE;
1483  }
1484 
1485  return NC_NOERR;
1486 }
1487 
1503 static int
1504 remove_coord_atts(hid_t hdf_datasetid)
1505 {
1506  htri_t attr_exists;
1507 
1508  /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1509  * attribute, delete it. */
1510  if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1511  return NC_EHDFERR;
1512  if (attr_exists)
1513  {
1514  if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1515  return NC_EHDFERR;
1516  }
1517 
1518  /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1519  if ((attr_exists = H5Aexists(hdf_datasetid,
1520  HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1521  return NC_EHDFERR;
1522  if (attr_exists)
1523  {
1524  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1525  return NC_EHDFERR;
1526  }
1527  if ((attr_exists = H5Aexists(hdf_datasetid,
1528  HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1529  return NC_EHDFERR;
1530  if (attr_exists)
1531  {
1532  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1533  return NC_EHDFERR;
1534  }
1535  return NC_NOERR;
1536 }
1537 
1552 static int
1553 write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1554 {
1555  NC_HDF5_GRP_INFO_T *hdf5_grp;
1556  NC_HDF5_VAR_INFO_T *hdf5_var;
1557  nc_bool_t replace_existing_var = NC_FALSE;
1558  int retval;
1559 
1560  assert(var && var->format_var_info && grp && grp->format_grp_info);
1561 
1562  LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1563 
1564  /* Get HDF5-specific group and var info. */
1565  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1566  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1567 
1568  /* If the variable has already been created & the fill value changed,
1569  * indicate that the existing variable should be replaced. */
1570  if (var->created && var->fill_val_changed)
1571  {
1572  replace_existing_var = NC_TRUE;
1573  var->fill_val_changed = NC_FALSE;
1574  /* If the variable is going to be replaced, we need to flag any
1575  other attributes associated with the variable as 'dirty', or
1576  else *only* the fill value attribute will be copied over and
1577  the rest will be lost. See
1578  https://github.com/Unidata/netcdf-c/issues/239 */
1579  flag_atts_dirty(var->att);
1580  }
1581 
1582  /* Is this a coordinate var that has already been created in
1583  * the HDF5 file as a dimscale dataset? Check for dims with the
1584  * same name in this group. If there is one, check to see if
1585  * this object exists in the HDF group. */
1586  if (var->became_coord_var)
1587  {
1588  if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1589  {
1590  nc_bool_t exists;
1591 
1592  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1593  return retval;
1594  if (exists)
1595  {
1596  /* Indicate that the variable already exists, and should
1597  * be replaced. */
1598  replace_existing_var = NC_TRUE;
1599  flag_atts_dirty(var->att);
1600  }
1601  }
1602  }
1603 
1604  /* Check dims if the variable will be replaced, so that the
1605  * dimensions will be de-attached and re-attached correctly. */
1606  if (replace_existing_var)
1607  {
1608  NC_DIM_INFO_T *d1;
1609 
1610  /* Is there a dim with this var's name? */
1611  if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1612  {
1613  nc_bool_t exists;
1614  assert(d1->format_dim_info && d1->hdr.name);
1615 
1616  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1617  return retval;
1618  if (exists)
1619  {
1620  hid_t dsid;
1621 
1622  /* Find dataset ID for dimension */
1623  if (d1->coord_var)
1624  dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1625  else
1626  dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1627  assert(dsid > 0);
1628 
1629  /* If we're replacing an existing dimscale dataset, go to
1630  * every var in the file and detach this dimension scale,
1631  * because we have to delete it. */
1632  if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1633  var->dimids[0], dsid)))
1634  return retval;
1635  }
1636  }
1637  }
1638 
1639  /* If this is not a dimension scale, remove any attached scales,
1640  * and delete dimscale attributes from the var. */
1641  if (var->was_coord_var && var->dimscale_attached)
1642  {
1643  int d;
1644 
1645  /* If the variable already exists in the file, Remove any dimension scale
1646  * attributes from it, if they exist. */
1647  if (var->created)
1648  if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1649  return retval;
1650 
1651  /* If this is a regular var, detach all its dim scales. */
1652  for (d = 0; d < var->ndims; d++)
1653  {
1654  if (var->dimscale_attached[d])
1655  {
1656  hid_t dsid; /* Dataset ID for dimension */
1657  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1658  var->dim[d]->format_dim_info);
1659 
1660  /* Find dataset ID for dimension */
1661  if (var->dim[d]->coord_var)
1662  dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1663  else
1664  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1665  assert(dsid > 0);
1666 
1667  /* Detach this dim scale. */
1668  if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1669  return NC_EHDFERR;
1670  var->dimscale_attached[d] = NC_FALSE;
1671  }
1672  }
1673  }
1674 
1675  /* Delete the HDF5 dataset that is to be replaced. */
1676  if (replace_existing_var)
1677  {
1678  /* Free the HDF5 dataset id. */
1679  if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1680  return NC_EHDFERR;
1681  hdf5_var->hdf_datasetid = 0;
1682 
1683  /* Now delete the variable. */
1684  if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1685  return NC_EDIMMETA;
1686  }
1687 
1688  /* Create the dataset. */
1689  if (var->is_new_var || replace_existing_var)
1690  {
1691  if ((retval = var_create_dataset(grp, var, write_dimid)))
1692  return retval;
1693  }
1694  else
1695  {
1696  if (write_dimid && var->ndims)
1697  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1698  var->dimids[0])))
1699  return retval;
1700  }
1701 
1702  if (replace_existing_var)
1703  {
1704  /* If this is a dimension scale, reattach the scale everywhere it
1705  * is used. (Recall that netCDF dimscales are always 1-D). */
1706  if(var->dimscale)
1707  {
1708  if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1709  var->dimids[0], hdf5_var->hdf_datasetid)))
1710  return retval;
1711  }
1712  /* If it's not a dimension scale, clear the dimscale attached flags,
1713  * so the dimensions are re-attached. */
1714  else
1715  {
1716  if (var->dimscale_attached)
1717  memset(var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1718  }
1719  }
1720 
1721  /* Clear coord. var state transition flags */
1722  var->was_coord_var = NC_FALSE;
1723  var->became_coord_var = NC_FALSE;
1724 
1725  /* Now check the attributes for this var. */
1726  if (var->attr_dirty)
1727  {
1728  /* Write attributes for this var. */
1729  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1730  return retval;
1731  var->attr_dirty = NC_FALSE;
1732  }
1733 
1734  return NC_NOERR;
1735 }
1736 
1750 int
1751 nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1752 {
1753  NC_HDF5_DIM_INFO_T *hdf5_dim;
1754  NC_HDF5_GRP_INFO_T *hdf5_grp;
1755  hid_t spaceid = -1, create_propid = -1;
1756  hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1757  char dimscale_wo_var[NC_MAX_NAME];
1758  int retval = NC_NOERR;
1759 
1760  LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1761 
1762  /* Sanity check */
1763  assert(!dim->coord_var);
1764 
1765  /* Get HDF5-specific dim and group info. */
1766  hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1767  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1768 
1769  /* Create a property list. */
1770  if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1771  BAIL(NC_EHDFERR);
1772 
1773  /* Turn off recording of times associated with this object. */
1774  if (H5Pset_obj_track_times(create_propid, 0) < 0)
1775  BAIL(NC_EHDFERR);
1776 
1777  /* Set size of dataset to size of dimension. */
1778  dims[0] = dim->len;
1779  max_dims[0] = dim->len;
1780 
1781  /* If this dimension scale is unlimited (i.e. it's an unlimited
1782  * dimension), then set up chunking, with a chunksize of 1. */
1783  if (dim->unlimited)
1784  {
1785  max_dims[0] = H5S_UNLIMITED;
1786  if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1787  BAIL(NC_EHDFERR);
1788  }
1789 
1790  /* Set up space. */
1791  if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1792  BAIL(NC_EHDFERR);
1793 
1794  /* Turn on creation-order tracking. */
1795  if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1796  H5P_CRT_ORDER_INDEXED) < 0)
1797  BAIL(NC_EHDFERR);
1798 
1799  /* Create the dataset that will be the dimension scale. */
1800  LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1801  dim->hdr.name));
1802  if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1803  H5T_IEEE_F32BE, spaceid,
1804  H5P_DEFAULT, create_propid,
1805  H5P_DEFAULT)) < 0)
1806  BAIL(NC_EHDFERR);
1807 
1808  /* Indicate that this is a scale. Also indicate that not
1809  * be shown to the user as a variable. It is hidden. It is
1810  * a DIM WITHOUT A VARIABLE! */
1811  sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1812  if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1813  BAIL(NC_EHDFERR);
1814 
1815  /* Since this dimension was created out of order, we cannot rely on
1816  * it getting the correct dimid on file open. We must assign it
1817  * explicitly. */
1818  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1819  BAIL(retval);
1820 
1821 exit:
1822  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1823  BAIL2(NC_EHDFERR);
1824  if (create_propid > 0 && H5Pclose(create_propid) < 0)
1825  BAIL2(NC_EHDFERR);
1826  return retval;
1827 }
1828 
1841 static int
1842 write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1843 {
1844  NC_HDF5_DIM_INFO_T *hdf5_dim;
1845  int retval = NC_NOERR;
1846 
1847  assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1848 
1849  /* Get HDF5-specific dim and group info. */
1850  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1851 
1852  /* If there's no dimscale dataset for this dim, create one,
1853  * and mark that it should be hidden from netCDF as a
1854  * variable. (That is, it should appear as a dimension
1855  * without an associated variable.) */
1856  if (!hdf5_dim->hdf_dimscaleid)
1857  if ((retval = nc4_create_dim_wo_var(dim)))
1858  BAIL(retval);
1859 
1860  /* Did we extend an unlimited dimension? */
1861  if (dim->extended)
1862  {
1863  NC_VAR_INFO_T *v1 = NULL;
1864 
1865  assert(dim->unlimited);
1866 
1867  /* If this is a dimension with an associated coordinate var,
1868  * then update the length of that coord var. */
1869  v1 = dim->coord_var;
1870  if (v1)
1871  {
1872  NC_HDF5_VAR_INFO_T *hdf5_v1;
1873  hsize_t *new_size;
1874  int d1;
1875 
1876  hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1877 
1878  /* Extend the dimension scale dataset to reflect the new
1879  * length of the dimension. */
1880  if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1881  BAIL(NC_ENOMEM);
1882  for (d1 = 0; d1 < v1->ndims; d1++)
1883  {
1884  assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1885  new_size[d1] = v1->dim[d1]->len;
1886  }
1887  if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1888  BAIL(NC_EHDFERR);
1889  free(new_size);
1890  }
1891  }
1892 
1893  /* If desired, write the secret dimid. This will be used instead of
1894  * the dimid that the dimension would otherwise receive based on
1895  * creation order. This can be necessary when dims and their
1896  * coordinate variables were created in different order. */
1897  if (write_dimid && hdf5_dim->hdf_dimscaleid)
1898  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1899  BAIL(retval);
1900 
1901 exit:
1902 
1903  return retval;
1904 }
1905 
1918 int
1919 nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1920 {
1921  NC_DIM_INFO_T *dim = NULL;
1922  NC_VAR_INFO_T *var = NULL;
1923  NC_GRP_INFO_T *child_grp = NULL;
1924  int coord_varid = -1;
1925  int var_index = 0;
1926  int dim_index = 0;
1927  int retval;
1928  int i;
1929 
1930  assert(grp && grp->hdr.name &&
1931  ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1932  LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1933  bad_coord_order));
1934 
1935  /* Write global attributes for this group. */
1936  if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1937  return retval;
1938 
1939  /* Set the pointers to the beginning of the list of dims & vars in this
1940  * group. */
1941  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1942  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1943 
1944  /* Because of HDF5 ordering the dims and vars have to be stored in
1945  * this way to ensure that the dims and coordinate vars come out in
1946  * the correct order. */
1947  while (dim || var)
1948  {
1949  nc_bool_t found_coord, wrote_coord;
1950 
1951  /* Write non-coord dims in order, stopping at the first one that
1952  * has an associated coord var. */
1953  for (found_coord = NC_FALSE; dim && !found_coord; )
1954  {
1955  if (!dim->coord_var)
1956  {
1957  if ((retval = write_dim(dim, grp, bad_coord_order)))
1958  return retval;
1959  }
1960  else
1961  {
1962  coord_varid = dim->coord_var->hdr.id;
1963  found_coord = NC_TRUE;
1964  }
1965  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1966  }
1967 
1968  /* Write each var. When we get to the coord var we are waiting
1969  * for (if any), then we break after writing it. */
1970  for (wrote_coord = NC_FALSE; var && !wrote_coord; )
1971  {
1972  if ((retval = write_var(var, grp, bad_coord_order)))
1973  return retval;
1974  if (found_coord && var->hdr.id == coord_varid)
1975  wrote_coord = NC_TRUE;
1976  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
1977  }
1978  } /* end while */
1979 
1980  /* Attach dimscales to vars in this group. */
1981  if ((retval = attach_dimscales(grp)))
1982  return retval;
1983 
1984  /* If there are any child groups, write their metadata. */
1985  for (i = 0; i < ncindexsize(grp->children); i++)
1986  {
1987  child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
1988  assert(child_grp);
1989  if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
1990  return retval;
1991  }
1992  return NC_NOERR;
1993 }
1994 
2004 int
2005 nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
2006 {
2007  NC_GRP_INFO_T *child_grp;
2008  NC_HDF5_GRP_INFO_T *hdf5_grp;
2009  NC_TYPE_INFO_T *type;
2010  int retval;
2011  int i;
2012 
2013  assert(grp && grp->hdr.name && grp->format_grp_info);
2014  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2015 
2016  /* Get HDF5-specific group info. */
2017  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
2018 
2019  /* Create the group in the HDF5 file if it doesn't exist. */
2020  if (!hdf5_grp->hdf_grpid)
2021  if ((retval = create_group(grp)))
2022  return retval;
2023 
2024  /* If this is the root group of a file with strict NC3 rules, write
2025  * an attribute. But don't leave the attribute open. */
2026  if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2027  if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2028  return retval;
2029 
2030  /* If there are any user-defined types, write them now. */
2031  for(i=0;i<ncindexsize(grp->type);i++) {
2032  type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2033  assert(type);
2034  if ((retval = commit_type(grp, type)))
2035  return retval;
2036  }
2037 
2038  /* If there are any child groups, write their groups and types. */
2039  for(i=0;i<ncindexsize(grp->children);i++) {
2040  if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2041  if ((retval = nc4_rec_write_groups_types(child_grp)))
2042  return retval;
2043  }
2044  return NC_NOERR;
2045 }
2046 
2060 int
2061 nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2062 {
2063  NC_GRP_INFO_T *g;
2064  NC_VAR_INFO_T *var;
2065  NC_DIM_INFO_T *dim;
2066  int retval = NC_NOERR;
2067  int i;
2068 
2069  assert(grp && grp->hdr.name);
2070  LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2071 
2072  /* Perform var dimscale match for child groups. */
2073  for (i = 0; i < ncindexsize(grp->children); i++)
2074  {
2075  g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2076  assert(g);
2077  if ((retval = nc4_rec_match_dimscales(g)))
2078  return retval;
2079  }
2080 
2081  /* Check all the vars in this group. If they have dimscale info,
2082  * try and find a dimension for them. */
2083  for (i = 0; i < ncindexsize(grp->vars); i++)
2084  {
2085  NC_HDF5_VAR_INFO_T *hdf5_var;
2086  int ndims;
2087  int d;
2088 
2089  /* Get pointer to var and to the HDF5-specific var info. */
2090  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2091  assert(var && var->format_var_info);
2092  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2093 
2094  /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2095  /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2096  (from the initial calloc) which is a legitimate dimid. The code does not
2097  distinquish this case from the dimscale case where the id might actually
2098  be defined.
2099  The original nc4_find_dim searched up the group tree looking for the given
2100  dimid in one of the dim lists associated with each ancestor group.
2101  I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2102  However, here that is incorrect because it will find the dimid 0 always
2103  (if any dimensions were defined). Except that when dimscale dimids have
2104  been defined, one or more of the values in var->dimids will have a
2105  legitimate value.
2106  The solution I choose is to modify nc4_var_list_add to initialize dimids to
2107  illegal values (-1). This is another example of the problems with dimscales.
2108  */
2109  ndims = var->ndims;
2110  for (d = 0; d < ndims; d++)
2111  {
2112  if (var->dim[d] == NULL) {
2113  nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2114  }
2115  /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2116  }
2117 
2118  /* Skip dimension scale variables */
2119  if (!var->dimscale)
2120  {
2121  int d;
2122  int j;
2123 
2124  /* Are there dimscales for this variable? */
2125  if (hdf5_var->dimscale_hdf5_objids)
2126  {
2127  for (d = 0; d < var->ndims; d++)
2128  {
2129  nc_bool_t finished = NC_FALSE;
2130  LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2131 
2132  /* Check this and parent groups. */
2133  for (g = grp; g && !finished; g = g->parent)
2134  {
2135  /* Check all dims in this group. */
2136  for (j = 0; j < ncindexsize(g->dim); j++)
2137  {
2138  /* Get the HDF5 specific dim info. */
2139  NC_HDF5_DIM_INFO_T *hdf5_dim;
2140  dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2141  assert(dim && dim->format_dim_info);
2142  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2143 
2144  /* Check for exact match of fileno/objid arrays
2145  * to find identical objects in HDF5 file. */
2146  if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2147  hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2148  hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2149  hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2150  {
2151  LOG((4, "%s: for dimension %d, found dim %s", __func__,
2152  d, dim->hdr.name));
2153  var->dimids[d] = dim->hdr.id;
2154  var->dim[d] = dim;
2155  finished = NC_TRUE;
2156  break;
2157  }
2158  } /* next dim */
2159  } /* next grp */
2160  LOG((5, "%s: dimid for this dimscale is %d", __func__,
2161  var->type_info->hdr.id));
2162  } /* next var->dim */
2163  }
2164  /* No dimscales for this var! Invent phony dimensions. */
2165  else
2166  {
2167  hid_t spaceid = 0;
2168  hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2169  int dataset_ndims;
2170 
2171  /* Find the space information for this dimension. */
2172  if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2173  return NC_EHDFERR;
2174 
2175  /* Get the len of each dim in the space. */
2176  if (var->ndims)
2177  {
2178  if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2179  return NC_ENOMEM;
2180  if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2181  {
2182  free(h5dimlen);
2183  return NC_ENOMEM;
2184  }
2185  if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2186  h5dimlenmax)) < 0) {
2187  free(h5dimlenmax);
2188  free(h5dimlen);
2189  return NC_EHDFERR;
2190  }
2191  if (dataset_ndims != var->ndims) {
2192  free(h5dimlenmax);
2193  free(h5dimlen);
2194  return NC_EHDFERR;
2195  }
2196  }
2197  else
2198  {
2199  /* Make sure it's scalar. */
2200  if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2201  return NC_EHDFERR;
2202  }
2203 
2204  /* Release the space object. */
2205  if (H5Sclose(spaceid) < 0) {
2206  free(h5dimlen);
2207  free(h5dimlenmax);
2208  return NC_EHDFERR;
2209  }
2210 
2211  /* Create a phony dimension for each dimension in the
2212  * dataset, unless there already is one the correct
2213  * size. */
2214  for (d = 0; d < var->ndims; d++)
2215  {
2216  int k;
2217  int match;
2218  /* Is there already a phony dimension of the correct size? */
2219  for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2220  if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2221  if ((dim->len == h5dimlen[d]) &&
2222  ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2223  (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2224  {match = k; break;}
2225  }
2226 
2227  /* Didn't find a phony dim? Then create one. */
2228  if (match < 0)
2229  {
2230  char phony_dim_name[NC_MAX_NAME + 1];
2231  sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2232  LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2233  if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2234  {
2235  free(h5dimlenmax);
2236  free(h5dimlen);
2237  return retval;
2238  }
2239  /* Create struct for HDF5-specific dim info. */
2240  if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2241  return NC_ENOMEM;
2242  if (h5dimlenmax[d] == H5S_UNLIMITED)
2243  dim->unlimited = NC_TRUE;
2244  }
2245 
2246  /* The variable must remember the dimid. */
2247  var->dimids[d] = dim->hdr.id;
2248  var->dim[d] = dim;
2249  } /* next dim */
2250 
2251  /* Free the memory we malloced. */
2252  free(h5dimlen);
2253  free(h5dimlenmax);
2254  }
2255  }
2256  }
2257 
2258  return retval;
2259 }
2260 
2273 int
2274 nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype, int *type_class)
2275 {
2276  int retval = NC_NOERR;
2277 
2278  LOG((4, "%s xtype: %d", __func__, xtype));
2279  assert(type_class);
2280 
2281  /* If this is an atomic type, the answer is easy. */
2282  if (xtype <= NC_STRING)
2283  {
2284  switch (xtype)
2285  {
2286  case NC_BYTE:
2287  case NC_UBYTE:
2288  case NC_SHORT:
2289  case NC_USHORT:
2290  case NC_INT:
2291  case NC_UINT:
2292  case NC_INT64:
2293  case NC_UINT64:
2294  /* NC_INT is class used for all integral types */
2295  *type_class = NC_INT;
2296  break;
2297 
2298  case NC_FLOAT:
2299  case NC_DOUBLE:
2300  /* NC_FLOAT is class used for all floating-point types */
2301  *type_class = NC_FLOAT;
2302  break;
2303 
2304  case NC_CHAR:
2305  *type_class = NC_CHAR;
2306  break;
2307 
2308  case NC_STRING:
2309  *type_class = NC_STRING;
2310  break;
2311 
2312  default:
2313  BAIL(NC_EBADTYPE);
2314  }
2315  }
2316  else
2317  {
2318  NC_TYPE_INFO_T *type;
2319 
2320  /* See if it's a used-defined type */
2321  if ((retval = nc4_find_type(h5, xtype, &type)))
2322  BAIL(retval);
2323  if (!type)
2324  BAIL(NC_EBADTYPE);
2325 
2326  *type_class = type->nc_type_class;
2327  }
2328 
2329 exit:
2330  return retval;
2331 }
2332 
2344 void
2345 reportobject(int uselog, hid_t id, unsigned int type)
2346 {
2347  char name[NC_HDF5_MAX_NAME];
2348  ssize_t len;
2349  const char* typename = NULL;
2350  long long printid = (long long)id;
2351 
2352  len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2353  if(len < 0) return;
2354  name[len] = '\0';
2355 
2356  switch (type) {
2357  case H5F_OBJ_FILE: typename = "File"; break;
2358  case H5F_OBJ_DATASET: typename = "Dataset"; break;
2359  case H5F_OBJ_GROUP: typename = "Group"; break;
2360  case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2361  case H5F_OBJ_ATTR:
2362  typename = "Attribute";
2363  len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2364  if(len < 0) len = 0;
2365  name[len] = '\0';
2366  break;
2367  default: typename = "<unknown>"; break;
2368  }
2369 #ifdef LOGGING
2370  if(uselog) {
2371  LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2372  } else
2373 #endif
2374  {
2375  fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2376  }
2377 
2378 }
2379 
2390 static void
2391 reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2392 {
2393  int t,i;
2394  ssize_t ocount;
2395  size_t maxobjs = -1;
2396  hid_t* idlist = NULL;
2397 
2398  /* Always report somehow */
2399 #ifdef LOGGING
2400  if(uselog)
2401  LOG((0,"\nReport: open objects on %lld",(long long)fid));
2402  else
2403 #endif
2404  fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2405  maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2406  if(idlist != NULL) free(idlist);
2407  idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2408  for(t=0;t<ntypes;t++) {
2409  unsigned int ot = otypes[t];
2410  ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2411  for(i=0;i<ocount;i++) {
2412  hid_t o = idlist[i];
2413  reportobject(uselog,o,ot);
2414  }
2415  }
2416  if(idlist != NULL) free(idlist);
2417 }
2418 
2427 void
2428 reportopenobjects(int uselog, hid_t fid)
2429 {
2430  unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2431  H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2432 
2433  reportopenobjectsT(uselog, fid ,5, OTYPES);
2434 }
2435 
2443 void
2444 showopenobjects5(NC_FILE_INFO_T* h5)
2445 {
2446  NC_HDF5_FILE_INFO_T *hdf5_info;
2447 
2448  assert(h5 && h5->format_file_info);
2449  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2450 
2451  fprintf(stderr,"===== begin showopenobjects =====\n");
2452  reportopenobjects(0,hdf5_info->hdfid);
2453  fprintf(stderr,"===== end showopenobjects =====\n");
2454  fflush(stderr);
2455 }
2456 
2465 void
2466 showopenobjects(int ncid)
2467 {
2468  NC_FILE_INFO_T* h5 = NULL;
2469 
2470  /* Find our metadata for this file. */
2471  if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2472  fprintf(stderr,"failed\n");
2473  else
2474  showopenobjects5(h5);
2475  fflush(stderr);
2476 }
2477 
2489 int
2490 NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2491 {
2492  if(H5get_libversion(major,minor,release) < 0)
2493  return NC_EHDFERR;
2494  return NC_NOERR;
2495 }
2496 
2507 int
2508 NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2509 {
2510  NC_HDF5_FILE_INFO_T *hdf5_info;
2511  int stat = NC_NOERR;
2512  unsigned super;
2513  hid_t plist = -1;
2514 
2515  assert(h5 && h5->format_file_info);
2516  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2517 
2518  if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2519  {stat = NC_EHDFERR; goto done;}
2520  if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2521  {stat = NC_EHDFERR; goto done;}
2522  if(idp) *idp = (int)super;
2523 done:
2524  if(plist >= 0) H5Pclose(plist);
2525  return stat;
2526 }
2527 
2528 static int NC4_get_strict_att(NC_FILE_INFO_T*);
2529 static int NC4_walk(hid_t, int*);
2530 
2555 int
2556 NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2557 {
2558  int stat;
2559  int isnc4 = 0;
2560  int count;
2561 
2562  /* Look for NC3_STRICT_ATT_NAME */
2563  isnc4 = NC4_get_strict_att(h5);
2564  if(isnc4 > 0)
2565  goto done;
2566  /* attribute did not exist */
2567  /* => last resort: walk the HDF5 file looking for markers */
2568  count = 0;
2569  stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2570  &count);
2571  if(stat != NC_NOERR)
2572  isnc4 = 0;
2573  else /* Threshold is at least two matches */
2574  isnc4 = (count >= 2);
2575 
2576 done:
2577  return isnc4;
2578 }
2579 
2588 static int
2589 NC4_get_strict_att(NC_FILE_INFO_T *h5)
2590 {
2591  hid_t grpid = -1;
2592  hid_t attid = -1;
2593 
2594  /* Get root group ID. */
2595  grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2596 
2597  /* Try to extract the NC3_STRICT_ATT_NAME attribute */
2598  attid = H5Aopen_name(grpid, NC3_STRICT_ATT_NAME);
2599  H5Aclose(attid);
2600  return attid;
2601 }
2602 
2612 static int
2613 NC4_walk(hid_t gid, int* countp)
2614 {
2615  int ncstat = NC_NOERR;
2616  int i,j,na;
2617  ssize_t len;
2618  hsize_t nobj;
2619  herr_t err;
2620  int otype;
2621  hid_t grpid, dsid;
2622  char name[NC_HDF5_MAX_NAME];
2623 
2624  /* walk group members of interest */
2625  err = H5Gget_num_objs(gid, &nobj);
2626  if(err < 0) return err;
2627 
2628  for(i = 0; i < nobj; i++) {
2629  /* Get name & kind of object in the group */
2630  len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2631  if(len < 0) return len;
2632 
2633  otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2634  switch(otype) {
2635  case H5G_GROUP:
2636  grpid = H5Gopen(gid,name);
2637  NC4_walk(grpid,countp);
2638  H5Gclose(grpid);
2639  break;
2640  case H5G_DATASET: /* variables */
2641  /* Check for phony_dim */
2642  if(strcmp(name,"phony_dim")==0)
2643  *countp = *countp + 1;
2644  dsid = H5Dopen(gid,name);
2645  na = H5Aget_num_attrs(dsid);
2646  for(j = 0; j < na; j++) {
2647  hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2648  if(aid >= 0) {
2649  const NC_reservedatt* ra;
2650  ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2651  if(len < 0) return len;
2652  /* Is this a netcdf-4 marker attribute */
2653  /* Is this a netcdf-4 marker attribute */
2654  ra = NC_findreserved(name);
2655  if(ra != NULL)
2656  *countp = *countp + 1;
2657  }
2658  H5Aclose(aid);
2659  }
2660  H5Dclose(dsid);
2661  break;
2662  default:/* ignore */
2663  break;
2664  }
2665  }
2666  return ncstat;
2667 }
2668 
2669 
2670 /**************************************************/
2671 /* Filter registration support */
2672 
2673 static int
2674 filterlookup(int id)
2675 {
2676  int i;
2677  if(filters == NULL)
2678  filters = nclistnew();
2679  for(i=0;i<nclistlength(filters);i++) {
2680  NC_FILTER_INFO* x = nclistget(filters,i);
2681  if(x != NULL && x->id == id) return i; /* return position */
2682  }
2683  return -1;
2684 }
2685 
2686 static void
2687 reclaiminfo(NC_FILTER_INFO* info)
2688 {
2689  if(info != NULL)
2690  nullfree(info->info);
2691  nullfree(info);
2692 }
2693 
2694 static int
2695 filterremove(int pos)
2696 {
2697  NC_FILTER_INFO* info = NULL;
2698  if(filters == NULL)
2699  filters = nclistnew();
2700  if(pos < 0 || pos >= nclistlength(filters))
2701  return NC_EINVAL;
2702  info = nclistget(filters,pos);
2703  reclaiminfo(info);
2704  nclistremove(filters,pos);
2705  return NC_NOERR;
2706 }
2707 
2708 static NC_FILTER_INFO*
2709 dupfilterinfo(NC_FILTER_INFO* info)
2710 {
2711  NC_FILTER_INFO* dup = NULL;
2712  if(info == NULL) goto fail;
2713  if(info->info == NULL) goto fail;
2714  if((dup = calloc(1,sizeof(NC_FILTER_INFO))) == NULL) goto fail;
2715  *dup = *info;
2716  if((dup->info = calloc(1,sizeof(H5Z_class2_t))) == NULL) goto fail;
2717  {
2718  H5Z_class2_t* h5dup = (H5Z_class2_t*)dup->info;
2719  H5Z_class2_t* h5info = (H5Z_class2_t*)info->info;
2720  *h5dup = *h5info;
2721  }
2722  return dup;
2723 fail:
2724  reclaiminfo(dup);
2725  return NULL;
2726 }
2727 
2728 int
2729 nc4_filter_action(int op, int format, int id, NC_FILTER_INFO* info)
2730 {
2731  int stat = NC_NOERR;
2732  H5Z_class2_t* h5filterinfo = NULL;
2733  herr_t herr;
2734  int pos = -1;
2735  NC_FILTER_INFO* dup = NULL;
2736 
2737  if(format != NC_FILTER_FORMAT_HDF5)
2738  {stat = NC_ENOTNC4; goto done;}
2739 
2740  switch (op) {
2741  case FILTER_REG: /* Ignore id argument */
2742  if(info == NULL || info->info == NULL)
2743  {stat = NC_EINVAL; goto done;}
2744  if(info->version != NC_FILTER_INFO_VERSION
2745  || info->format != NC_FILTER_FORMAT_HDF5)
2746  {stat = NC_ENOTNC4; goto done;}
2747  h5filterinfo = info->info;
2748  /* Another sanity check */
2749  if(info->id != h5filterinfo->id)
2750  {stat = NC_EINVAL; goto done;}
2751  /* See if this filter is already defined */
2752  if((pos = filterlookup(id)) >= 0)
2753  {stat = NC_ENAMEINUSE; goto done;} /* Already defined */
2754  if((herr = H5Zregister(h5filterinfo)) < 0)
2755  {stat = NC_EFILTER; goto done;}
2756  /* Save a copy of the passed in info */
2757  if((dup = dupfilterinfo(info)) == NULL)
2758  {stat = NC_ENOMEM; goto done;}
2759  nclistpush(filters,dup);
2760  break;
2761  case FILTER_UNREG:
2762  if(id <= 0)
2763  {stat = NC_ENOTNC4; goto done;}
2764  /* See if this filter is already defined */
2765  if((pos = filterlookup(id)) < 0)
2766  {stat = NC_EFILTER; goto done;} /* Not defined */
2767  if((herr = H5Zunregister(id)) < 0)
2768  {stat = NC_EFILTER; goto done;}
2769  if((stat=filterremove(pos))) goto done;
2770  break;
2771  case FILTER_INQ:
2772  if(id <= 0)
2773  {stat = NC_ENOTNC4; goto done;}
2774  /* Look up the id in our local table */
2775  if((pos = filterlookup(id)) < 0)
2776  {stat = NC_EFILTER; goto done;} /* Not defined */
2777  if(info != NULL) {
2778  *info = *((NC_FILTER_INFO*)nclistget(filters,pos));
2779  }
2780  break;
2781  default:
2782  {stat = NC_EINTERNAL; goto done;}
2783  }
2784 done:
2785  return stat;
2786 }
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:405
#define NC_ENOTNC4
Attempting netcdf-4 operation on netcdf-3 file.
Definition: netcdf.h:448
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:138
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:276
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:438
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
#define NC_EINTERNAL
NetCDF Library Internal Error.
Definition: netcdf.h:431
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45
#define NC_STRING
string
Definition: netcdf.h:47
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
#define H5Z_FILTER_SZIP
ID of HDF SZIP filter.
Definition: dvarinq.c:16
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
size_t len
Length of VL data (in base type units)
Definition: netcdf.h:688
#define NC_ENAMEINUSE
String match to name in use.
Definition: netcdf.h:364
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:286
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:444
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:442
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:470
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:367
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:443
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:335
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:287
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:275
void * p
Pointer to VL data.
Definition: netcdf.h:689
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Free memory in a VLEN object.
Definition: dvlen.c:41
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:445
This is the type of arrays of vlens.
Definition: netcdf.h:687
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:285
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:379
#define NC_EPERM
Write to read only.
Definition: netcdf.h:336
#define NC_NOERR
No Error.
Definition: netcdf.h:325
#define NC_ENUM
enum types
Definition: netcdf.h:55
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:248
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46

Return to the Main Unidata NetCDF page.
Generated on Thu Nov 21 2019 04:31:58 for NetCDF. NetCDF is a Unidata library.