NetCDF  4.7.1
nc4hdf.c
Go to the documentation of this file.
1 /* Copyright 2018, University Corporation for Atmospheric
2  * Research. See the COPYRIGHT file for copying and redistribution
3  * conditions. */
17 #include "config.h"
18 #include "hdf5internal.h"
19 #include <math.h>
20 
21 #ifdef HAVE_INTTYPES_H
22 #define __STDC_FORMAT_MACROS
23 #include <inttypes.h>
24 #endif
25 
26 #define NC_HDF5_MAX_NAME 1024
28 /* WARNING: GLOBAL VARIABLE */
29 
30 /* Define list of registered filters */
31 static NClist* filters = NULL;
32 
41 static int
42 flag_atts_dirty(NCindex *attlist) {
43 
44  NC_ATT_INFO_T *att = NULL;
45  int i;
46 
47  if(attlist == NULL) {
48  return NC_NOERR;
49  }
50 
51  for(i=0;i<ncindexsize(attlist);i++) {
52  att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
53  if(att == NULL) continue;
54  att->dirty = NC_TRUE;
55  }
56 
57  return NC_NOERR;
58 }
59 
76 int
77 rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
78 {
79  NC_VAR_INFO_T *var;
80  NC_GRP_INFO_T *child_grp;
81  int d, i;
82  int retval;
83 
84  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
85  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
86 
87  /* If there are any child groups, attach dimscale there, if needed. */
88  for (i = 0; i < ncindexsize(grp->children); i++)
89  {
90  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
91  assert(child_grp);
92  if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
93  return retval;
94  }
95 
96  /* Find any vars that use this dimension id. */
97  for (i = 0; i < ncindexsize(grp->vars); i++)
98  {
99  NC_HDF5_VAR_INFO_T *hdf5_var;
100 
101  var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
102  assert(var && var->format_var_info);
103  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
104 
105  for (d = 0; d < var->ndims; d++)
106  {
107  if (var->dimids[d] == dimid && !var->dimscale)
108  {
109  LOG((2, "%s: attaching scale for dimid %d to var %s",
110  __func__, var->dimids[d], var->hdr.name));
111  if (var->created)
112  {
113  if (H5DSattach_scale(hdf5_var->hdf_datasetid,
114  dimscaleid, d) < 0)
115  return NC_EHDFERR;
116  var->dimscale_attached[d] = NC_TRUE;
117  }
118  }
119  }
120  }
121  return NC_NOERR;
122 }
123 
140 int
141 rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
142 {
143  NC_VAR_INFO_T *var;
144  NC_GRP_INFO_T *child_grp;
145  int d, i;
146  int retval;
147 
148  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
149  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
150 
151  /* If there are any child groups, detach dimscale there, if needed. */
152  for(i=0;i<ncindexsize(grp->children);i++) {
153  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
154  if(child_grp == NULL) continue;
155  if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
156  return retval;
157  }
158 
159  /* Find any vars that use this dimension id. */
160  for (i = 0; i < ncindexsize(grp->vars); i++)
161  {
162  NC_HDF5_VAR_INFO_T *hdf5_var;
163  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
164  assert(var && var->format_var_info);
165  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
166 
167  for (d = 0; d < var->ndims; d++)
168  {
169  if (var->dimids[d] == dimid && !var->dimscale)
170  {
171  LOG((2, "%s: detaching scale for dimid %d to var %s",
172  __func__, var->dimids[d], var->hdr.name));
173  if (var->created)
174  {
175  if (var->dimscale_attached && var->dimscale_attached[d])
176  {
177  if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
178  dimscaleid, d) < 0)
179  return NC_EHDFERR;
180  var->dimscale_attached[d] = NC_FALSE;
181  }
182  }
183  }
184  }
185  }
186  return NC_NOERR;
187 }
188 
200 int
201 nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
202 {
203  NC_VAR_INFO_T *var;
204  NC_HDF5_VAR_INFO_T *hdf5_var;
205 
206  assert(grp && grp->format_grp_info && dataset);
207 
208  /* Find the requested varid. */
209  if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
210  return NC_ENOTVAR;
211  assert(var && var->hdr.id == varid && var->format_var_info);
212  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
213 
214  /* Open this dataset if necessary. */
215  if (!hdf5_var->hdf_datasetid)
216  {
217  NC_HDF5_GRP_INFO_T *hdf5_grp;
218  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
219 
220  if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
221  var->hdr.name, H5P_DEFAULT)) < 0)
222  return NC_ENOTVAR;
223  }
224 
225  *dataset = hdf5_var->hdf_datasetid;
226 
227  return NC_NOERR;
228 }
229 
241 int
242 nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
243 {
244  size_t size;
245  int retval;
246 
247  /* Find out how much space we need for this type's fill value. */
248  if (var->type_info->nc_type_class == NC_VLEN)
249  size = sizeof(nc_vlen_t);
250  else if (var->type_info->nc_type_class == NC_STRING)
251  size = sizeof(char *);
252  else
253  {
254  if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size)))
255  return retval;
256  }
257  assert(size);
258 
259  /* Allocate the space. */
260  if (!((*fillp) = calloc(1, size)))
261  return NC_ENOMEM;
262 
263  /* If the user has set a fill_value for this var, use, otherwise
264  * find the default fill value. */
265  if (var->fill_value)
266  {
267  LOG((4, "Found a fill value for var %s", var->hdr.name));
268  if (var->type_info->nc_type_class == NC_VLEN)
269  {
270  nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
271  size_t basetypesize = 0;
272 
273  if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
274  return retval;
275 
276  fv_vlen->len = in_vlen->len;
277  if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
278  {
279  free(*fillp);
280  *fillp = NULL;
281  return NC_ENOMEM;
282  }
283  memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
284  }
285  else if (var->type_info->nc_type_class == NC_STRING)
286  {
287  if (*(char **)var->fill_value)
288  if (!(**(char ***)fillp = strdup(*(char **)var->fill_value)))
289  {
290  free(*fillp);
291  *fillp = NULL;
292  return NC_ENOMEM;
293  }
294  }
295  else
296  memcpy((*fillp), var->fill_value, size);
297  }
298  else
299  {
300  if (nc4_get_default_fill_value(var->type_info, *fillp))
301  {
302  /* Note: release memory, but don't return error on failure */
303  free(*fillp);
304  *fillp = NULL;
305  }
306  }
307 
308  return NC_NOERR;
309 }
310 
328 int
329 nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
330  hid_t *hdf_typeid, int endianness)
331 {
332  NC_TYPE_INFO_T *type;
333  hid_t typeid = 0;
334  int retval = NC_NOERR;
335 
336  assert(hdf_typeid && h5);
337 
338  *hdf_typeid = -1;
339 
340  /* Determine an appropriate HDF5 datatype */
341  if (xtype == NC_NAT)
342  return NC_EBADTYPE;
343  else if (xtype == NC_CHAR || xtype == NC_STRING)
344  {
345  /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
346  if (xtype == NC_CHAR)
347  {
348  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
349  return NC_EHDFERR;
350  if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
351  BAIL(NC_EVARMETA);
352  if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
353  BAIL(NC_EVARMETA);
354 
355  /* Take ownership of the newly created HDF5 datatype */
356  *hdf_typeid = typeid;
357  typeid = 0;
358  }
359  else
360  {
361  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
362  return NC_EHDFERR;
363  if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
364  BAIL(NC_EVARMETA);
365  if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
366  BAIL(NC_EVARMETA);
367 
368  /* Take ownership of the newly created HDF5 datatype */
369  *hdf_typeid = typeid;
370  typeid = 0;
371  }
372  }
373  else
374  {
375  /* All other types use an existing HDF5 datatype */
376  switch (xtype)
377  {
378  case NC_BYTE: /* signed 1 byte integer */
379  if (endianness == NC_ENDIAN_LITTLE)
380  typeid = H5T_STD_I8LE;
381  else if (endianness == NC_ENDIAN_BIG)
382  typeid = H5T_STD_I8BE;
383  else
384  typeid = H5T_NATIVE_SCHAR;
385  break;
386 
387  case NC_SHORT: /* signed 2 byte integer */
388  if (endianness == NC_ENDIAN_LITTLE)
389  typeid = H5T_STD_I16LE;
390  else if (endianness == NC_ENDIAN_BIG)
391  typeid = H5T_STD_I16BE;
392  else
393  typeid = H5T_NATIVE_SHORT;
394  break;
395 
396  case NC_INT:
397  if (endianness == NC_ENDIAN_LITTLE)
398  typeid = H5T_STD_I32LE;
399  else if (endianness == NC_ENDIAN_BIG)
400  typeid = H5T_STD_I32BE;
401  else
402  typeid = H5T_NATIVE_INT;
403  break;
404 
405  case NC_UBYTE:
406  if (endianness == NC_ENDIAN_LITTLE)
407  typeid = H5T_STD_U8LE;
408  else if (endianness == NC_ENDIAN_BIG)
409  typeid = H5T_STD_U8BE;
410  else
411  typeid = H5T_NATIVE_UCHAR;
412  break;
413 
414  case NC_USHORT:
415  if (endianness == NC_ENDIAN_LITTLE)
416  typeid = H5T_STD_U16LE;
417  else if (endianness == NC_ENDIAN_BIG)
418  typeid = H5T_STD_U16BE;
419  else
420  typeid = H5T_NATIVE_USHORT;
421  break;
422 
423  case NC_UINT:
424  if (endianness == NC_ENDIAN_LITTLE)
425  typeid = H5T_STD_U32LE;
426  else if (endianness == NC_ENDIAN_BIG)
427  typeid = H5T_STD_U32BE;
428  else
429  typeid = H5T_NATIVE_UINT;
430  break;
431 
432  case NC_INT64:
433  if (endianness == NC_ENDIAN_LITTLE)
434  typeid = H5T_STD_I64LE;
435  else if (endianness == NC_ENDIAN_BIG)
436  typeid = H5T_STD_I64BE;
437  else
438  typeid = H5T_NATIVE_LLONG;
439  break;
440 
441  case NC_UINT64:
442  if (endianness == NC_ENDIAN_LITTLE)
443  typeid = H5T_STD_U64LE;
444  else if (endianness == NC_ENDIAN_BIG)
445  typeid = H5T_STD_U64BE;
446  else
447  typeid = H5T_NATIVE_ULLONG;
448  break;
449 
450  case NC_FLOAT:
451  if (endianness == NC_ENDIAN_LITTLE)
452  typeid = H5T_IEEE_F32LE;
453  else if (endianness == NC_ENDIAN_BIG)
454  typeid = H5T_IEEE_F32BE;
455  else
456  typeid = H5T_NATIVE_FLOAT;
457  break;
458 
459  case NC_DOUBLE:
460  if (endianness == NC_ENDIAN_LITTLE)
461  typeid = H5T_IEEE_F64LE;
462  else if (endianness == NC_ENDIAN_BIG)
463  typeid = H5T_IEEE_F64BE;
464  else
465  typeid = H5T_NATIVE_DOUBLE;
466  break;
467 
468  default:
469  /* Maybe this is a user defined type? */
470  if (nc4_find_type(h5, xtype, &type))
471  return NC_EBADTYPE;
472  if (!type)
473  return NC_EBADTYPE;
474  typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
475  break;
476  }
477  assert(typeid);
478 
479  /* Copy the HDF5 datatype, so the function operates uniformly */
480  if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
481  return NC_EHDFERR;
482  typeid = 0;
483  }
484  assert(*hdf_typeid != -1);
485 
486 exit:
487  if (typeid > 0 && H5Tclose(typeid) < 0)
488  BAIL2(NC_EHDFERR);
489  return retval;
490 }
491 
506 static int
507 put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
508 {
509  NC_HDF5_GRP_INFO_T *hdf5_grp;
510  hid_t datasetid = 0, locid;
511  hid_t attid = 0, spaceid = 0, file_typeid = 0;
512  hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
513  hsize_t dims[1]; /* netcdf attributes always 1-D. */
514  htri_t attr_exists;
515  int reuse_att = 0; /* Will be true if we can re-use an existing att. */
516  void *data;
517  int phoney_data = 99;
518  int retval = NC_NOERR;
519 
520  assert(att->hdr.name && grp && grp->format_grp_info);
521  LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
522  "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
523  att->nc_typeid, att->len));
524 
525  /* Get HDF5-specific group info. */
526  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
527 
528  /* If the file is read-only, return an error. */
529  if (grp->nc4_info->no_write)
530  BAIL(NC_EPERM);
531 
532  /* Get the hid to attach the attribute to, or read it from. */
533  if (varid == NC_GLOBAL)
534  locid = hdf5_grp->hdf_grpid;
535  else
536  {
537  if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
538  BAIL(retval);
539  locid = datasetid;
540  }
541 
542  /* Get the length ready, and find the HDF type we'll be
543  * writing. */
544  dims[0] = att->len;
545  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
546  &file_typeid, 0)))
547  BAIL(retval);
548 
549  /* Even if the length is zero, HDF5 won't let me write with a
550  * NULL pointer. So if the length of the att is zero, point to
551  * some phoney data (which won't be written anyway.)*/
552  if (!dims[0])
553  data = &phoney_data;
554  else if (att->data)
555  data = att->data;
556  else if (att->stdata)
557  data = att->stdata;
558  else
559  data = att->vldata;
560 
561  /* NC_CHAR types require some extra work. The space ID is set to
562  * scalar, and the type is told how long the string is. If it's
563  * really zero length, set the size to 1. (The fact that it's
564  * really zero will be marked by the NULL dataspace, but HDF5
565  * doesn't allow me to set the size of the type to zero.)*/
566  if (att->nc_typeid == NC_CHAR)
567  {
568  size_t string_size = dims[0];
569  if (!string_size)
570  {
571  string_size = 1;
572  if ((spaceid = H5Screate(H5S_NULL)) < 0)
573  BAIL(NC_EATTMETA);
574  }
575  else
576  {
577  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
578  BAIL(NC_EATTMETA);
579  }
580  if (H5Tset_size(file_typeid, string_size) < 0)
581  BAIL(NC_EATTMETA);
582  if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
583  BAIL(NC_EATTMETA);
584  }
585  else
586  {
587  if (!att->len)
588  {
589  if ((spaceid = H5Screate(H5S_NULL)) < 0)
590  BAIL(NC_EATTMETA);
591  }
592  else
593  {
594  if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
595  BAIL(NC_EATTMETA);
596  }
597  }
598 
599  /* Does the att exists already? */
600  if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
601  BAIL(NC_EHDFERR);
602  if (attr_exists)
603  {
604  hssize_t npoints;
605 
606  /* Open the attribute. */
607  if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
608  BAIL(NC_EATTMETA);
609 
610  /* Find the type of the existing attribute. */
611  if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
612  BAIL(NC_EATTMETA);
613 
614  /* How big is the attribute? */
615  if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
616  BAIL(NC_EATTMETA);
617  if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
618  BAIL(NC_EATTMETA);
619 
620  /* Delete the attribute. */
621  if (file_typeid != existing_att_typeid || npoints != att->len)
622  {
623  if (H5Adelete(locid, att->hdr.name) < 0)
624  BAIL(NC_EHDFERR);
625  }
626  else
627  {
628  reuse_att++;
629  }
630  }
631 
632  /* Create the attribute. */
633  if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid,
634  H5P_DEFAULT)) < 0)
635  BAIL(NC_EATTMETA);
636 
637  /* Write the values, (even if length is zero). */
638  if (H5Awrite(attid, file_typeid, data) < 0)
639  BAIL(NC_EATTMETA);
640 
641 exit:
642  if (file_typeid && H5Tclose(file_typeid))
643  BAIL2(NC_EHDFERR);
644  if (attid > 0 && H5Aclose(attid) < 0)
645  BAIL2(NC_EHDFERR);
646  if (existing_att_typeid && H5Tclose(existing_att_typeid))
647  BAIL2(NC_EHDFERR);
648  if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
649  BAIL2(NC_EHDFERR);
650  if (spaceid > 0 && H5Sclose(spaceid) < 0)
651  BAIL2(NC_EHDFERR);
652  if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
653  BAIL2(NC_EHDFERR);
654  return retval;
655 }
656 
668 static int
669 write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
670 {
671  NC_ATT_INFO_T *att;
672  int retval;
673  int i;
674 
675  for(i = 0; i < ncindexsize(attlist); i++)
676  {
677  att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
678  assert(att);
679  if (att->dirty)
680  {
681  LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
682  if ((retval = put_att_grpa(grp, varid, att)))
683  return retval;
684  att->dirty = NC_FALSE;
685  att->created = NC_TRUE;
686  }
687  }
688  return NC_NOERR;
689 }
690 
704 static int
705 write_coord_dimids(NC_VAR_INFO_T *var)
706 {
707  NC_HDF5_VAR_INFO_T *hdf5_var;
708  hsize_t coords_len[1];
709  hid_t c_spaceid = -1, c_attid = -1;
710  int retval = NC_NOERR;
711 
712  assert(var && var->format_var_info);
713 
714  /* Get HDF5-specific var info. */
715  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
716 
717  /* Set up space for attribute. */
718  coords_len[0] = var->ndims;
719  if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
720  BAIL(NC_EHDFERR);
721 
722  /* Create the attribute. */
723  if ((c_attid = H5Acreate(hdf5_var->hdf_datasetid, COORDINATES,
724  H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
725  BAIL(NC_EHDFERR);
726 
727  /* Write our attribute. */
728  if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
729  BAIL(NC_EHDFERR);
730 
731 exit:
732  if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
733  BAIL2(NC_EHDFERR);
734  if (c_attid >= 0 && H5Aclose(c_attid) < 0)
735  BAIL2(NC_EHDFERR);
736  return retval;
737 }
738 
749 static int
750 write_netcdf4_dimid(hid_t datasetid, int dimid)
751 {
752  hid_t dimid_spaceid = -1, dimid_attid = -1;
753  htri_t attr_exists;
754  int retval = NC_NOERR;
755 
756  /* Create the space. */
757  if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
758  BAIL(NC_EHDFERR);
759 
760  /* Does the attribute already exist? If so, don't try to create it. */
761  if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
762  BAIL(NC_EHDFERR);
763  if (attr_exists)
764  dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
765  H5P_DEFAULT, H5P_DEFAULT);
766  else
767  /* Create the attribute if needed. */
768  dimid_attid = H5Acreate(datasetid, NC_DIMID_ATT_NAME,
769  H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
770  if (dimid_attid < 0)
771  BAIL(NC_EHDFERR);
772 
773 
774  /* Write it. */
775  LOG((4, "%s: writing secret dimid %d", __func__, dimid));
776  if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
777  BAIL(NC_EHDFERR);
778 
779 exit:
780  /* Close stuff*/
781  if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
782  BAIL2(NC_EHDFERR);
783  if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
784  BAIL2(NC_EHDFERR);
785 
786  return retval;
787 }
788 
803 static int
804 var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
805 {
806  NC_HDF5_GRP_INFO_T *hdf5_grp;
807  NC_HDF5_VAR_INFO_T *hdf5_var;
808  hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
809  hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
810  int d;
811  void *fillp = NULL;
812  NC_DIM_INFO_T *dim = NULL;
813  char *name_to_use;
814  int retval;
815 
816  assert(grp && grp->format_grp_info && var && var->format_var_info);
817 
818  LOG((3, "%s:: name %s", __func__, var->hdr.name));
819 
820  /* Get HDF5-specific group and var info. */
821  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
822  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
823 
824  /* Scalar or not, we need a creation property list. */
825  if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
826  BAIL(NC_EHDFERR);
827  if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
828  BAIL(NC_EHDFERR);
829 
830  /* Turn off object tracking times in HDF5. */
831  if (H5Pset_obj_track_times(plistid, 0) < 0)
832  BAIL(NC_EHDFERR);
833 
834  /* Find the HDF5 type of the dataset. */
835  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
836  var->type_info->endianness)))
837  BAIL(retval);
838 
839  /* Figure out what fill value to set, if any. */
840  if (var->no_fill)
841  {
842  /* Required to truly turn HDF5 fill values off */
843  if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
844  BAIL(NC_EHDFERR);
845  }
846  else
847  {
848  if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
849  BAIL(retval);
850 
851  /* If there is a fill value, set it. */
852  if (fillp)
853  {
854  if (var->type_info->nc_type_class == NC_STRING)
855  {
856  if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
857  BAIL(NC_EHDFERR);
858  }
859  else
860  {
861  /* The fill value set in HDF5 must always be presented as
862  * a native type, even if the endianness for this dataset
863  * is non-native. HDF5 will translate the fill value to
864  * the target endiannesss. */
865  hid_t fill_typeid = 0;
866 
867  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
869  BAIL(retval);
870  if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
871  {
872  if (H5Tclose(fill_typeid) < 0)
873  BAIL(NC_EHDFERR);
874  BAIL(NC_EHDFERR);
875  }
876  if (H5Tclose(fill_typeid) < 0)
877  BAIL(NC_EHDFERR);
878  }
879  }
880  }
881 
882  /* If the user wants to shuffle the data, set that up now. */
883  if (var->shuffle) {
884  if (H5Pset_shuffle(plistid) < 0)
885  BAIL(NC_EHDFERR);
886  }
887 
888  /* If the user wants to deflate the data, set that up now. */
889  if (var->deflate) {
890  if (H5Pset_deflate(plistid, var->deflate_level) < 0)
891  BAIL(NC_EHDFERR);
892  } else if(var->filterid) {
893  /* Handle szip case here */
894  if(var->filterid == H5Z_FILTER_SZIP) {
895  int options_mask;
896  int bits_per_pixel;
897  if(var->nparams != 2)
898  BAIL(NC_EFILTER);
899  options_mask = (int)var->params[0];
900  bits_per_pixel = (int)var->params[1];
901  if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
902  BAIL(NC_EFILTER);
903  } else {
904  herr_t code = H5Pset_filter(plistid, var->filterid, H5Z_FLAG_MANDATORY, var->nparams, var->params);
905  if(code < 0) {
906  BAIL(NC_EFILTER);
907  }
908  }
909  }
910 
911  /* If the user wants to fletcher error correcton, set that up now. */
912  if (var->fletcher32)
913  if (H5Pset_fletcher32(plistid) < 0)
914  BAIL(NC_EHDFERR);
915 
916  /* If ndims non-zero, get info for all dimensions. We look up the
917  dimids and get the len of each dimension. We need this to create
918  the space for the dataset. In netCDF a dimension length of zero
919  means an unlimited dimension. */
920  if (var->ndims)
921  {
922  int unlimdim = 0;
923 
924  /* Check to see if any unlimited dimensions are used in this var. */
925  for (d = 0; d < var->ndims; d++) {
926  dim = var->dim[d];
927  assert(dim && dim->hdr.id == var->dimids[d]);
928  if (dim->unlimited)
929  unlimdim++;
930  }
931 
932  /* If there are no unlimited dims, and no filters, and the user
933  * has not specified chunksizes, use contiguous variable for
934  * better performance. */
935  if (!var->shuffle && !var->deflate && !var->fletcher32 &&
936  (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
937  var->contiguous = NC_TRUE;
938 
939  /* Gather current & maximum dimension sizes, along with chunk sizes */
940  for (d = 0; d < var->ndims; d++)
941  {
942  dim = var->dim[d];
943  assert(dim && dim->hdr.id == var->dimids[d]);
944  dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
945  maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
946  if (!var->contiguous) {
947  if (var->chunksizes[d])
948  chunksize[d] = var->chunksizes[d];
949  else
950  {
951  size_t type_size;
952  if (var->type_info->nc_type_class == NC_STRING)
953  type_size = sizeof(char *);
954  else
955  type_size = var->type_info->size;
956 
957  /* Unlimited dim always gets chunksize of 1. */
958  if (dim->unlimited)
959  chunksize[d] = 1;
960  else
961  chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
962  1/(double)(var->ndims - unlimdim));
963 
964  /* If the chunksize is greater than the dim
965  * length, make it the dim length. */
966  if (!dim->unlimited && chunksize[d] > dim->len)
967  chunksize[d] = dim->len;
968 
969  /* Remember the computed chunksize */
970  var->chunksizes[d] = chunksize[d];
971  }
972  }
973  }
974 
975  if (var->contiguous)
976  {
977  if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
978  BAIL(NC_EHDFERR);
979  }
980  else
981  {
982  if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
983  BAIL(NC_EHDFERR);
984  }
985 
986  /* Create the dataspace. */
987  if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
988  BAIL(NC_EHDFERR);
989  }
990  else
991  {
992  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
993  BAIL(NC_EHDFERR);
994  }
995 
996  /* Turn on creation order tracking. */
997  if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
998  H5P_CRT_ORDER_INDEXED) < 0)
999  BAIL(NC_EHDFERR);
1000 
1001  /* Set per-var chunk cache, for chunked datasets. */
1002  if (!var->contiguous && var->chunk_cache_size)
1003  if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
1004  var->chunk_cache_size, var->chunk_cache_preemption) < 0)
1005  BAIL(NC_EHDFERR);
1006 
1007  /* At long last, create the dataset. */
1008  name_to_use = var->hdf5_name ? var->hdf5_name : var->hdr.name;
1009  LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
1010  name_to_use, typeid));
1011  if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
1012  spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
1013  BAIL(NC_EHDFERR);
1014  var->created = NC_TRUE;
1015  var->is_new_var = NC_FALSE;
1016 
1017  /* Always write the hidden coordinates attribute, which lists the
1018  * dimids of this var. When present, this speeds opens. When no
1019  * present, dimscale matching is used. */
1020  if (var->ndims > 1)
1021  if ((retval = write_coord_dimids(var)))
1022  BAIL(retval);
1023 
1024  /* If this is a dimscale, mark it as such in the HDF5 file. Also
1025  * find the dimension info and store the dataset id of the dimscale
1026  * dataset. */
1027  if (var->dimscale)
1028  {
1029  if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1030  BAIL(NC_EHDFERR);
1031 
1032  /* If this is a multidimensional coordinate variable, write a
1033  * coordinates attribute. */
1034  /* if (var->ndims > 1) */
1035  /* if ((retval = write_coord_dimids(var))) */
1036  /* BAIL(retval); */
1037 
1038  /* If desired, write the netCDF dimid. */
1039  if (write_dimid)
1040  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1041  BAIL(retval);
1042  }
1043 
1044  /* Write attributes for this var. */
1045  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1046  BAIL(retval);
1047  var->attr_dirty = NC_FALSE;
1048 
1049 exit:
1050  if (typeid > 0 && H5Tclose(typeid) < 0)
1051  BAIL2(NC_EHDFERR);
1052  if (plistid > 0 && H5Pclose(plistid) < 0)
1053  BAIL2(NC_EHDFERR);
1054  if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1055  BAIL2(NC_EHDFERR);
1056  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1057  BAIL2(NC_EHDFERR);
1058  if (fillp)
1059  {
1060  if (var->type_info->nc_type_class == NC_VLEN)
1061  nc_free_vlen((nc_vlen_t *)fillp);
1062  else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1063  free(*(char **)fillp);
1064  free(fillp);
1065  }
1066 
1067  return retval;
1068 }
1069 
1080 int
1081 nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1082 {
1083  size_t chunk_size_bytes = 1;
1084  int d;
1085  int retval;
1086 
1087  /* Nothing to be done. */
1088  if (var->contiguous)
1089  return NC_NOERR;
1090 #ifdef USE_PARALLEL4
1091  return NC_NOERR;
1092 #endif
1093 
1094  /* How many bytes in the chunk? */
1095  for (d = 0; d < var->ndims; d++)
1096  chunk_size_bytes *= var->chunksizes[d];
1097  if (var->type_info->size)
1098  chunk_size_bytes *= var->type_info->size;
1099  else
1100  chunk_size_bytes *= sizeof(char *);
1101 
1102  /* If the chunk cache is too small, and the user has not changed
1103  * the default value of the chunk cache size, then increase the
1104  * size of the cache. */
1105  if (var->chunk_cache_size == CHUNK_CACHE_SIZE)
1106  if (chunk_size_bytes > var->chunk_cache_size)
1107  {
1108  var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1109  if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE)
1110  var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE;
1111  if ((retval = nc4_reopen_dataset(grp, var)))
1112  return retval;
1113  }
1114 
1115  return NC_NOERR;
1116 }
1117 
1133 static int
1134 commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1135 {
1136  NC_HDF5_GRP_INFO_T *hdf5_grp;
1137  NC_HDF5_TYPE_INFO_T *hdf5_type;
1138  hid_t base_hdf_typeid;
1139  int retval;
1140 
1141  assert(grp && grp->format_grp_info && type && type->format_type_info);
1142 
1143  /* Get HDF5-specific group and type info. */
1144  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1145  hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1146 
1147  /* Did we already record this type? */
1148  if (type->committed)
1149  return NC_NOERR;
1150 
1151  /* Is this a compound type? */
1152  if (type->nc_type_class == NC_COMPOUND)
1153  {
1154  NC_FIELD_INFO_T *field;
1155  hid_t hdf_base_typeid, hdf_typeid;
1156  int i;
1157 
1158  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1159  return NC_EHDFERR;
1160  LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1161  hdf5_type->hdf_typeid));
1162 
1163  for(i=0;i<nclistlength(type->u.c.field);i++)
1164  {
1165  field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1166  assert(field);
1167  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1168  &hdf_base_typeid, type->endianness)))
1169  return retval;
1170 
1171  /* If this is an array, create a special array type. */
1172  if (field->ndims)
1173  {
1174  int d;
1175  hsize_t dims[NC_MAX_VAR_DIMS];
1176 
1177  for (d = 0; d < field->ndims; d++)
1178  dims[d] = field->dim_size[d];
1179  if ((hdf_typeid = H5Tarray_create(hdf_base_typeid, field->ndims,
1180  dims, NULL)) < 0)
1181  {
1182  if (H5Tclose(hdf_base_typeid) < 0)
1183  return NC_EHDFERR;
1184  return NC_EHDFERR;
1185  }
1186  if (H5Tclose(hdf_base_typeid) < 0)
1187  return NC_EHDFERR;
1188  }
1189  else
1190  hdf_typeid = hdf_base_typeid;
1191  LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1192  field->offset, hdf_typeid));
1193  if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1194  hdf_typeid) < 0)
1195  return NC_EHDFERR;
1196  if (H5Tclose(hdf_typeid) < 0)
1197  return NC_EHDFERR;
1198  }
1199  }
1200  else if (type->nc_type_class == NC_VLEN)
1201  {
1202  /* Find the HDF typeid of the base type of this vlen. */
1203  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1204  &base_hdf_typeid, type->endianness)))
1205  return retval;
1206 
1207  /* Create a vlen type. */
1208  if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1209  return NC_EHDFERR;
1210  }
1211  else if (type->nc_type_class == NC_OPAQUE)
1212  {
1213  /* Create the opaque type. */
1214  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1215  return NC_EHDFERR;
1216  }
1217  else if (type->nc_type_class == NC_ENUM)
1218  {
1219  NC_ENUM_MEMBER_INFO_T *enum_m;
1220  int i;
1221 
1222  if (nclistlength(type->u.e.enum_member) == 0)
1223  return NC_EINVAL;
1224 
1225  /* Find the HDF typeid of the base type of this enum. */
1226  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1227  &base_hdf_typeid, type->endianness)))
1228  return retval;
1229 
1230  /* Create an enum type. */
1231  if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1232  return NC_EHDFERR;
1233 
1234  /* Add all the members to the HDF5 type. */
1235  for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1236  enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1237  if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1238  return NC_EHDFERR;
1239  }
1240  }
1241  else
1242  {
1243  LOG((0, "Unknown class: %d", type->nc_type_class));
1244  return NC_EBADTYPE;
1245  }
1246 
1247  /* Commit the type. */
1248  if (H5Tcommit(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1249  return NC_EHDFERR;
1250  type->committed = NC_TRUE;
1251  LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1252  hdf5_type->hdf_typeid));
1253 
1254  /* Later we will always use the native typeid. In this case, it is
1255  * a copy of the same type pointed to by hdf_typeid, but it's
1256  * easier to maintain a copy. */
1257  if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1258  H5T_DIR_DEFAULT)) < 0)
1259  return NC_EHDFERR;
1260 
1261  return NC_NOERR;
1262 }
1263 
1274 static int
1275 write_nc3_strict_att(hid_t hdf_grpid)
1276 {
1277  hid_t attid = 0, spaceid = 0;
1278  int one = 1;
1279  int retval = NC_NOERR;
1280  htri_t attr_exists;
1281 
1282  /* If the attribute already exists, call that a success and return
1283  * NC_NOERR. */
1284  if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1285  return NC_EHDFERR;
1286  if (attr_exists)
1287  return NC_NOERR;
1288 
1289  /* Create the attribute to mark this as a file that needs to obey
1290  * strict netcdf-3 rules. */
1291  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1292  BAIL(NC_EFILEMETA);
1293  if ((attid = H5Acreate(hdf_grpid, NC3_STRICT_ATT_NAME,
1294  H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1295  BAIL(NC_EFILEMETA);
1296  if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1297  BAIL(NC_EFILEMETA);
1298 
1299 exit:
1300  if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1301  BAIL2(NC_EFILEMETA);
1302  if (attid > 0 && (H5Aclose(attid) < 0))
1303  BAIL2(NC_EFILEMETA);
1304  return retval;
1305 }
1306 
1319 static int
1320 create_group(NC_GRP_INFO_T *grp)
1321 {
1322  NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1323  hid_t gcpl_id = -1;
1324  int retval = NC_NOERR;;
1325 
1326  assert(grp && grp->format_grp_info && grp->parent &&
1327  grp->parent->format_grp_info);
1328 
1329  /* Get HDF5 specific group info for group and parent. */
1330  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1331  parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1332  assert(parent_hdf5_grp->hdf_grpid);
1333 
1334  /* Create group, with link_creation_order set in the group
1335  * creation property list. */
1336  if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1337  BAIL(NC_EHDFERR);
1338 
1339  /* Set track_times to be FALSE. */
1340  if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1341  BAIL(NC_EHDFERR);
1342 
1343  /* Tell HDF5 to keep track of objects in creation order. */
1344  if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1345  BAIL(NC_EHDFERR);
1346 
1347  /* Tell HDF5 to keep track of attributes in creation order. */
1348  if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1349  BAIL(NC_EHDFERR);
1350 
1351  /* Create the group. */
1352  if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1353  H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1354  BAIL(NC_EHDFERR);
1355 
1356 exit:
1357  if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1358  BAIL2(NC_EHDFERR);
1359  if (retval)
1360  if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1361  BAIL2(NC_EHDFERR);
1362  return retval;
1363 }
1364 
1376 static int
1377 attach_dimscales(NC_GRP_INFO_T *grp)
1378 {
1379  NC_VAR_INFO_T *var;
1380  NC_HDF5_VAR_INFO_T *hdf5_var;
1381  int d, v;
1382 
1383  /* Attach dimension scales. */
1384  for (v = 0; v < ncindexsize(grp->vars); v++)
1385  {
1386  /* Get pointer to var and HDF5-specific var info. */
1387  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1388  assert(var && var->format_var_info);
1389  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1390 
1391  /* Scales themselves do not attach. But I really wish they
1392  * would. */
1393  if (var->dimscale)
1394  continue;
1395 
1396  /* Find the scale for each dimension, if any, and attach it. */
1397  for (d = 0; d < var->ndims; d++)
1398  {
1399  /* Is there a dimscale for this dimension? */
1400  if (var->dimscale_attached)
1401  {
1402  if (!var->dimscale_attached[d])
1403  {
1404  hid_t dsid; /* Dataset ID for dimension */
1405  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1406  var->dim[d]->format_dim_info);
1407 
1408  LOG((2, "%s: attaching scale for dimid %d to var %s",
1409  __func__, var->dimids[d], var->hdr.name));
1410 
1411  /* Find dataset ID for dimension */
1412  if (var->dim[d]->coord_var)
1413  dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1414  else
1415  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1416  assert(dsid > 0);
1417 
1418  /* Attach the scale. */
1419  if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1420  return NC_EHDFERR;
1421  var->dimscale_attached[d] = NC_TRUE;
1422  }
1423  }
1424  }
1425  }
1426 
1427  return NC_NOERR;
1428 }
1429 
1440 static int
1441 var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1442 {
1443  htri_t link_exists;
1444 
1445  /* Reset the boolean */
1446  *exists = NC_FALSE;
1447 
1448  /* Check if the object name exists in the group */
1449  if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1450  return NC_EHDFERR;
1451  if (link_exists)
1452  {
1453  H5G_stat_t statbuf;
1454 
1455  /* Get info about the object */
1456  if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1457  return NC_EHDFERR;
1458 
1459  if (H5G_DATASET == statbuf.type)
1460  *exists = NC_TRUE;
1461  }
1462 
1463  return NC_NOERR;
1464 }
1465 
1481 static int
1482 remove_coord_atts(hid_t hdf_datasetid)
1483 {
1484  htri_t attr_exists;
1485 
1486  /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1487  * attribute, delete it. */
1488  if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1489  return NC_EHDFERR;
1490  if (attr_exists)
1491  {
1492  if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1493  return NC_EHDFERR;
1494  }
1495 
1496  /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1497  if ((attr_exists = H5Aexists(hdf_datasetid,
1498  HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1499  return NC_EHDFERR;
1500  if (attr_exists)
1501  {
1502  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1503  return NC_EHDFERR;
1504  }
1505  if ((attr_exists = H5Aexists(hdf_datasetid,
1506  HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1507  return NC_EHDFERR;
1508  if (attr_exists)
1509  {
1510  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1511  return NC_EHDFERR;
1512  }
1513  return NC_NOERR;
1514 }
1515 
1530 static int
1531 write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1532 {
1533  NC_HDF5_GRP_INFO_T *hdf5_grp;
1534  NC_HDF5_VAR_INFO_T *hdf5_var;
1535  nc_bool_t replace_existing_var = NC_FALSE;
1536  int retval;
1537 
1538  assert(var && var->format_var_info && grp && grp->format_grp_info);
1539 
1540  LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1541 
1542  /* Get HDF5-specific group and var info. */
1543  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1544  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1545 
1546  /* If the variable has already been created & the fill value changed,
1547  * indicate that the existing variable should be replaced. */
1548  if (var->created && var->fill_val_changed)
1549  {
1550  replace_existing_var = NC_TRUE;
1551  var->fill_val_changed = NC_FALSE;
1552  /* If the variable is going to be replaced, we need to flag any
1553  other attributes associated with the variable as 'dirty', or
1554  else *only* the fill value attribute will be copied over and
1555  the rest will be lost. See
1556  https://github.com/Unidata/netcdf-c/issues/239 */
1557  flag_atts_dirty(var->att);
1558  }
1559 
1560  /* Is this a coordinate var that has already been created in
1561  * the HDF5 file as a dimscale dataset? Check for dims with the
1562  * same name in this group. If there is one, check to see if
1563  * this object exists in the HDF group. */
1564  if (var->became_coord_var)
1565  {
1566  if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1567  {
1568  nc_bool_t exists;
1569 
1570  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1571  return retval;
1572  if (exists)
1573  {
1574  /* Indicate that the variable already exists, and should
1575  * be replaced. */
1576  replace_existing_var = NC_TRUE;
1577  flag_atts_dirty(var->att);
1578  }
1579  }
1580  }
1581 
1582  /* Check dims if the variable will be replaced, so that the
1583  * dimensions will be de-attached and re-attached correctly. */
1584  if (replace_existing_var)
1585  {
1586  NC_DIM_INFO_T *d1;
1587 
1588  /* Is there a dim with this var's name? */
1589  if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1590  {
1591  nc_bool_t exists;
1592  assert(d1->format_dim_info && d1->hdr.name);
1593 
1594  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1595  return retval;
1596  if (exists)
1597  {
1598  hid_t dsid;
1599 
1600  /* Find dataset ID for dimension */
1601  if (d1->coord_var)
1602  dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1603  else
1604  dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1605  assert(dsid > 0);
1606 
1607  /* If we're replacing an existing dimscale dataset, go to
1608  * every var in the file and detach this dimension scale,
1609  * because we have to delete it. */
1610  if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1611  var->dimids[0], dsid)))
1612  return retval;
1613  }
1614  }
1615  }
1616 
1617  /* If this is not a dimension scale, remove any attached scales,
1618  * and delete dimscale attributes from the var. */
1619  if (var->was_coord_var && var->dimscale_attached)
1620  {
1621  int d;
1622 
1623  /* If the variable already exists in the file, Remove any dimension scale
1624  * attributes from it, if they exist. */
1625  if (var->created)
1626  if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1627  return retval;
1628 
1629  /* If this is a regular var, detach all its dim scales. */
1630  for (d = 0; d < var->ndims; d++)
1631  {
1632  if (var->dimscale_attached[d])
1633  {
1634  hid_t dsid; /* Dataset ID for dimension */
1635  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1636  var->dim[d]->format_dim_info);
1637 
1638  /* Find dataset ID for dimension */
1639  if (var->dim[d]->coord_var)
1640  dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1641  else
1642  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1643  assert(dsid > 0);
1644 
1645  /* Detach this dim scale. */
1646  if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1647  return NC_EHDFERR;
1648  var->dimscale_attached[d] = NC_FALSE;
1649  }
1650  }
1651  }
1652 
1653  /* Delete the HDF5 dataset that is to be replaced. */
1654  if (replace_existing_var)
1655  {
1656  /* Free the HDF5 dataset id. */
1657  if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1658  return NC_EHDFERR;
1659  hdf5_var->hdf_datasetid = 0;
1660 
1661  /* Now delete the variable. */
1662  if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1663  return NC_EDIMMETA;
1664  }
1665 
1666  /* Create the dataset. */
1667  if (var->is_new_var || replace_existing_var)
1668  {
1669  if ((retval = var_create_dataset(grp, var, write_dimid)))
1670  return retval;
1671  }
1672  else
1673  {
1674  if (write_dimid && var->ndims)
1675  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1676  var->dimids[0])))
1677  return retval;
1678  }
1679 
1680  if (replace_existing_var)
1681  {
1682  /* If this is a dimension scale, reattach the scale everywhere it
1683  * is used. (Recall that netCDF dimscales are always 1-D). */
1684  if(var->dimscale)
1685  {
1686  if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1687  var->dimids[0], hdf5_var->hdf_datasetid)))
1688  return retval;
1689  }
1690  /* If it's not a dimension scale, clear the dimscale attached flags,
1691  * so the dimensions are re-attached. */
1692  else
1693  {
1694  if (var->dimscale_attached)
1695  memset(var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1696  }
1697  }
1698 
1699  /* Clear coord. var state transition flags */
1700  var->was_coord_var = NC_FALSE;
1701  var->became_coord_var = NC_FALSE;
1702 
1703  /* Now check the attributes for this var. */
1704  if (var->attr_dirty)
1705  {
1706  /* Write attributes for this var. */
1707  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1708  return retval;
1709  var->attr_dirty = NC_FALSE;
1710  }
1711 
1712  return NC_NOERR;
1713 }
1714 
1728 int
1729 nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1730 {
1731  NC_HDF5_DIM_INFO_T *hdf5_dim;
1732  NC_HDF5_GRP_INFO_T *hdf5_grp;
1733  hid_t spaceid = -1, create_propid = -1;
1734  hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1735  char dimscale_wo_var[NC_MAX_NAME];
1736  int retval = NC_NOERR;
1737 
1738  LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1739 
1740  /* Sanity check */
1741  assert(!dim->coord_var);
1742 
1743  /* Get HDF5-specific dim and group info. */
1744  hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1745  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1746 
1747  /* Create a property list. */
1748  if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1749  BAIL(NC_EHDFERR);
1750 
1751  /* Turn off recording of times associated with this object. */
1752  if (H5Pset_obj_track_times(create_propid, 0) < 0)
1753  BAIL(NC_EHDFERR);
1754 
1755  /* Set size of dataset to size of dimension. */
1756  dims[0] = dim->len;
1757  max_dims[0] = dim->len;
1758 
1759  /* If this dimension scale is unlimited (i.e. it's an unlimited
1760  * dimension), then set up chunking, with a chunksize of 1. */
1761  if (dim->unlimited)
1762  {
1763  max_dims[0] = H5S_UNLIMITED;
1764  if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1765  BAIL(NC_EHDFERR);
1766  }
1767 
1768  /* Set up space. */
1769  if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1770  BAIL(NC_EHDFERR);
1771 
1772  /* Turn on creation-order tracking. */
1773  if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1774  H5P_CRT_ORDER_INDEXED) < 0)
1775  BAIL(NC_EHDFERR);
1776 
1777  /* Create the dataset that will be the dimension scale. */
1778  LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1779  dim->hdr.name));
1780  if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1781  H5T_IEEE_F32BE, spaceid,
1782  H5P_DEFAULT, create_propid,
1783  H5P_DEFAULT)) < 0)
1784  BAIL(NC_EHDFERR);
1785 
1786  /* Indicate that this is a scale. Also indicate that not
1787  * be shown to the user as a variable. It is hidden. It is
1788  * a DIM WITHOUT A VARIABLE! */
1789  sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1790  if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1791  BAIL(NC_EHDFERR);
1792 
1793  /* Since this dimension was created out of order, we cannot rely on
1794  * it getting the correct dimid on file open. We must assign it
1795  * explicitly. */
1796  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1797  BAIL(retval);
1798 
1799 exit:
1800  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1801  BAIL2(NC_EHDFERR);
1802  if (create_propid > 0 && H5Pclose(create_propid) < 0)
1803  BAIL2(NC_EHDFERR);
1804  return retval;
1805 }
1806 
1819 static int
1820 write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1821 {
1822  NC_HDF5_DIM_INFO_T *hdf5_dim;
1823  int retval = NC_NOERR;
1824 
1825  assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1826 
1827  /* Get HDF5-specific dim and group info. */
1828  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1829 
1830  /* If there's no dimscale dataset for this dim, create one,
1831  * and mark that it should be hidden from netCDF as a
1832  * variable. (That is, it should appear as a dimension
1833  * without an associated variable.) */
1834  if (!hdf5_dim->hdf_dimscaleid)
1835  if ((retval = nc4_create_dim_wo_var(dim)))
1836  BAIL(retval);
1837 
1838  /* Did we extend an unlimited dimension? */
1839  if (dim->extended)
1840  {
1841  NC_VAR_INFO_T *v1 = NULL;
1842 
1843  assert(dim->unlimited);
1844  /* If this is a dimension without a variable, then update
1845  * the secret length information at the end of the NAME
1846  * attribute. */
1847  v1 = (NC_VAR_INFO_T *)ncindexlookup(grp->vars, dim->hdr.name);
1848  if (v1)
1849  {
1850  NC_HDF5_VAR_INFO_T *hdf5_v1;
1851  hsize_t *new_size;
1852  int d1;
1853 
1854  hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1855 
1856  /* Extend the dimension scale dataset to reflect the new
1857  * length of the dimension. */
1858  if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1859  BAIL(NC_ENOMEM);
1860  for (d1 = 0; d1 < v1->ndims; d1++)
1861  {
1862  assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1863  new_size[d1] = v1->dim[d1]->len;
1864  }
1865  if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1866  BAIL(NC_EHDFERR);
1867  free(new_size);
1868  }
1869  }
1870 
1871  /* If desired, write the secret dimid. This will be used instead of
1872  * the dimid that the dimension would otherwise receive based on
1873  * creation order. This can be necessary when dims and their
1874  * coordinate variables were created in different order. */
1875  if (write_dimid && hdf5_dim->hdf_dimscaleid)
1876  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1877  BAIL(retval);
1878 
1879 exit:
1880 
1881  return retval;
1882 }
1883 
1896 int
1897 nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1898 {
1899  NC_DIM_INFO_T *dim = NULL;
1900  NC_VAR_INFO_T *var = NULL;
1901  NC_GRP_INFO_T *child_grp = NULL;
1902  int coord_varid = -1;
1903  int var_index = 0;
1904  int dim_index = 0;
1905  int retval;
1906  int i;
1907 
1908  assert(grp && grp->hdr.name &&
1909  ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1910  LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1911  bad_coord_order));
1912 
1913  /* Write global attributes for this group. */
1914  if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1915  return retval;
1916 
1917  /* Set the pointers to the beginning of the list of dims & vars in this
1918  * group. */
1919  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1920  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1921 
1922  /* Because of HDF5 ordering the dims and vars have to be stored in
1923  * this way to ensure that the dims and coordinate vars come out in
1924  * the correct order. */
1925  while (dim || var)
1926  {
1927  nc_bool_t found_coord, wrote_coord;
1928 
1929  /* Write non-coord dims in order, stopping at the first one that
1930  * has an associated coord var. */
1931  for (found_coord = NC_FALSE; dim && !found_coord; )
1932  {
1933  if (!dim->coord_var)
1934  {
1935  if ((retval = write_dim(dim, grp, bad_coord_order)))
1936  return retval;
1937  }
1938  else
1939  {
1940  coord_varid = dim->coord_var->hdr.id;
1941  found_coord = NC_TRUE;
1942  }
1943  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1944  }
1945 
1946  /* Write each var. When we get to the coord var we are waiting
1947  * for (if any), then we break after writing it. */
1948  for (wrote_coord = NC_FALSE; var && !wrote_coord; )
1949  {
1950  if ((retval = write_var(var, grp, bad_coord_order)))
1951  return retval;
1952  if (found_coord && var->hdr.id == coord_varid)
1953  wrote_coord = NC_TRUE;
1954  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
1955  }
1956  } /* end while */
1957 
1958  /* Attach dimscales to vars in this group. */
1959  if ((retval = attach_dimscales(grp)))
1960  return retval;
1961 
1962  /* If there are any child groups, write their metadata. */
1963  for (i = 0; i < ncindexsize(grp->children); i++)
1964  {
1965  child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
1966  assert(child_grp);
1967  if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
1968  return retval;
1969  }
1970  return NC_NOERR;
1971 }
1972 
1982 int
1983 nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
1984 {
1985  NC_GRP_INFO_T *child_grp;
1986  NC_HDF5_GRP_INFO_T *hdf5_grp;
1987  NC_TYPE_INFO_T *type;
1988  int retval;
1989  int i;
1990 
1991  assert(grp && grp->hdr.name && grp->format_grp_info);
1992  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
1993 
1994  /* Get HDF5-specific group info. */
1995  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1996 
1997  /* Create the group in the HDF5 file if it doesn't exist. */
1998  if (!hdf5_grp->hdf_grpid)
1999  if ((retval = create_group(grp)))
2000  return retval;
2001 
2002  /* If this is the root group of a file with strict NC3 rules, write
2003  * an attribute. But don't leave the attribute open. */
2004  if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2005  if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2006  return retval;
2007 
2008  /* If there are any user-defined types, write them now. */
2009  for(i=0;i<ncindexsize(grp->type);i++) {
2010  type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2011  assert(type);
2012  if ((retval = commit_type(grp, type)))
2013  return retval;
2014  }
2015 
2016  /* If there are any child groups, write their groups and types. */
2017  for(i=0;i<ncindexsize(grp->children);i++) {
2018  if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2019  if ((retval = nc4_rec_write_groups_types(child_grp)))
2020  return retval;
2021  }
2022  return NC_NOERR;
2023 }
2024 
2038 int
2039 nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2040 {
2041  NC_GRP_INFO_T *g;
2042  NC_VAR_INFO_T *var;
2043  NC_DIM_INFO_T *dim;
2044  int retval = NC_NOERR;
2045  int i;
2046 
2047  assert(grp && grp->hdr.name);
2048  LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2049 
2050  /* Perform var dimscale match for child groups. */
2051  for (i = 0; i < ncindexsize(grp->children); i++)
2052  {
2053  g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2054  assert(g);
2055  if ((retval = nc4_rec_match_dimscales(g)))
2056  return retval;
2057  }
2058 
2059  /* Check all the vars in this group. If they have dimscale info,
2060  * try and find a dimension for them. */
2061  for (i = 0; i < ncindexsize(grp->vars); i++)
2062  {
2063  NC_HDF5_VAR_INFO_T *hdf5_var;
2064  int ndims;
2065  int d;
2066 
2067  /* Get pointer to var and to the HDF5-specific var info. */
2068  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2069  assert(var && var->format_var_info);
2070  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2071 
2072  /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2073  /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2074  (from the initial calloc) which is a legitimate dimid. The code does not
2075  distinquish this case from the dimscale case where the id might actually
2076  be defined.
2077  The original nc4_find_dim searched up the group tree looking for the given
2078  dimid in one of the dim lists associated with each ancestor group.
2079  I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2080  However, here that is incorrect because it will find the dimid 0 always
2081  (if any dimensions were defined). Except that when dimscale dimids have
2082  been defined, one or more of the values in var->dimids will have a
2083  legitimate value.
2084  The solution I choose is to modify nc4_var_list_add to initialize dimids to
2085  illegal values (-1). This is another example of the problems with dimscales.
2086  */
2087  ndims = var->ndims;
2088  for (d = 0; d < ndims; d++)
2089  {
2090  if (var->dim[d] == NULL) {
2091  nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2092  }
2093  /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2094  }
2095 
2096  /* Skip dimension scale variables */
2097  if (!var->dimscale)
2098  {
2099  int d;
2100  int j;
2101 
2102  /* Are there dimscales for this variable? */
2103  if (hdf5_var->dimscale_hdf5_objids)
2104  {
2105  for (d = 0; d < var->ndims; d++)
2106  {
2107  nc_bool_t finished = NC_FALSE;
2108  LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2109 
2110  /* Check this and parent groups. */
2111  for (g = grp; g && !finished; g = g->parent)
2112  {
2113  /* Check all dims in this group. */
2114  for (j = 0; j < ncindexsize(g->dim); j++)
2115  {
2116  /* Get the HDF5 specific dim info. */
2117  NC_HDF5_DIM_INFO_T *hdf5_dim;
2118  dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2119  assert(dim && dim->format_dim_info);
2120  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2121 
2122  /* Check for exact match of fileno/objid arrays
2123  * to find identical objects in HDF5 file. */
2124  if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2125  hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2126  hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2127  hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2128  {
2129  LOG((4, "%s: for dimension %d, found dim %s", __func__,
2130  d, dim->hdr.name));
2131  var->dimids[d] = dim->hdr.id;
2132  var->dim[d] = dim;
2133  finished = NC_TRUE;
2134  break;
2135  }
2136  } /* next dim */
2137  } /* next grp */
2138  LOG((5, "%s: dimid for this dimscale is %d", __func__,
2139  var->type_info->hdr.id));
2140  } /* next var->dim */
2141  }
2142  /* No dimscales for this var! Invent phony dimensions. */
2143  else
2144  {
2145  hid_t spaceid = 0;
2146  hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2147  int dataset_ndims;
2148 
2149  /* Find the space information for this dimension. */
2150  if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2151  return NC_EHDFERR;
2152 
2153  /* Get the len of each dim in the space. */
2154  if (var->ndims)
2155  {
2156  if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2157  return NC_ENOMEM;
2158  if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2159  {
2160  free(h5dimlen);
2161  return NC_ENOMEM;
2162  }
2163  if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2164  h5dimlenmax)) < 0) {
2165  free(h5dimlenmax);
2166  free(h5dimlen);
2167  return NC_EHDFERR;
2168  }
2169  if (dataset_ndims != var->ndims) {
2170  free(h5dimlenmax);
2171  free(h5dimlen);
2172  return NC_EHDFERR;
2173  }
2174  }
2175  else
2176  {
2177  /* Make sure it's scalar. */
2178  if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2179  return NC_EHDFERR;
2180  }
2181 
2182  /* Release the space object. */
2183  if (H5Sclose(spaceid) < 0) {
2184  free(h5dimlen);
2185  free(h5dimlenmax);
2186  return NC_EHDFERR;
2187  }
2188 
2189  /* Create a phony dimension for each dimension in the
2190  * dataset, unless there already is one the correct
2191  * size. */
2192  for (d = 0; d < var->ndims; d++)
2193  {
2194  int k;
2195  int match;
2196  /* Is there already a phony dimension of the correct size? */
2197  for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2198  if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2199  if ((dim->len == h5dimlen[d]) &&
2200  ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2201  (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2202  {match = k; break;}
2203  }
2204 
2205  /* Didn't find a phony dim? Then create one. */
2206  if (match < 0)
2207  {
2208  char phony_dim_name[NC_MAX_NAME + 1];
2209  sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2210  LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2211  if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2212  {
2213  free(h5dimlenmax);
2214  free(h5dimlen);
2215  return retval;
2216  }
2217  /* Create struct for HDF5-specific dim info. */
2218  if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2219  return NC_ENOMEM;
2220  if (h5dimlenmax[d] == H5S_UNLIMITED)
2221  dim->unlimited = NC_TRUE;
2222  }
2223 
2224  /* The variable must remember the dimid. */
2225  var->dimids[d] = dim->hdr.id;
2226  var->dim[d] = dim;
2227  } /* next dim */
2228 
2229  /* Free the memory we malloced. */
2230  free(h5dimlen);
2231  free(h5dimlenmax);
2232  }
2233  }
2234  }
2235 
2236  return retval;
2237 }
2238 
2251 int
2252 nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype, int *type_class)
2253 {
2254  int retval = NC_NOERR;
2255 
2256  LOG((4, "%s xtype: %d", __func__, xtype));
2257  assert(type_class);
2258 
2259  /* If this is an atomic type, the answer is easy. */
2260  if (xtype <= NC_STRING)
2261  {
2262  switch (xtype)
2263  {
2264  case NC_BYTE:
2265  case NC_UBYTE:
2266  case NC_SHORT:
2267  case NC_USHORT:
2268  case NC_INT:
2269  case NC_UINT:
2270  case NC_INT64:
2271  case NC_UINT64:
2272  /* NC_INT is class used for all integral types */
2273  *type_class = NC_INT;
2274  break;
2275 
2276  case NC_FLOAT:
2277  case NC_DOUBLE:
2278  /* NC_FLOAT is class used for all floating-point types */
2279  *type_class = NC_FLOAT;
2280  break;
2281 
2282  case NC_CHAR:
2283  *type_class = NC_CHAR;
2284  break;
2285 
2286  case NC_STRING:
2287  *type_class = NC_STRING;
2288  break;
2289 
2290  default:
2291  BAIL(NC_EBADTYPE);
2292  }
2293  }
2294  else
2295  {
2296  NC_TYPE_INFO_T *type;
2297 
2298  /* See if it's a used-defined type */
2299  if ((retval = nc4_find_type(h5, xtype, &type)))
2300  BAIL(retval);
2301  if (!type)
2302  BAIL(NC_EBADTYPE);
2303 
2304  *type_class = type->nc_type_class;
2305  }
2306 
2307 exit:
2308  return retval;
2309 }
2310 
2322 void
2323 reportobject(int uselog, hid_t id, unsigned int type)
2324 {
2325  char name[NC_HDF5_MAX_NAME];
2326  ssize_t len;
2327  const char* typename = NULL;
2328  long long printid = (long long)id;
2329 
2330  len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2331  if(len < 0) return;
2332  name[len] = '\0';
2333 
2334  switch (type) {
2335  case H5F_OBJ_FILE: typename = "File"; break;
2336  case H5F_OBJ_DATASET: typename = "Dataset"; break;
2337  case H5F_OBJ_GROUP: typename = "Group"; break;
2338  case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2339  case H5F_OBJ_ATTR:
2340  typename = "Attribute";
2341  len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2342  if(len < 0) len = 0;
2343  name[len] = '\0';
2344  break;
2345  default: typename = "<unknown>"; break;
2346  }
2347 #ifdef LOGGING
2348  if(uselog) {
2349  LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2350  } else
2351 #endif
2352  {
2353  fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2354  }
2355 
2356 }
2357 
2368 static void
2369 reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2370 {
2371  int t,i;
2372  ssize_t ocount;
2373  size_t maxobjs = -1;
2374  hid_t* idlist = NULL;
2375 
2376  /* Always report somehow */
2377 #ifdef LOGGING
2378  if(uselog)
2379  LOG((0,"\nReport: open objects on %lld",(long long)fid));
2380  else
2381 #endif
2382  fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2383  maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2384  if(idlist != NULL) free(idlist);
2385  idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2386  for(t=0;t<ntypes;t++) {
2387  unsigned int ot = otypes[t];
2388  ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2389  for(i=0;i<ocount;i++) {
2390  hid_t o = idlist[i];
2391  reportobject(uselog,o,ot);
2392  }
2393  }
2394  if(idlist != NULL) free(idlist);
2395 }
2396 
2405 void
2406 reportopenobjects(int uselog, hid_t fid)
2407 {
2408  unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2409  H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2410 
2411  reportopenobjectsT(uselog, fid ,5, OTYPES);
2412 }
2413 
2421 void
2422 showopenobjects5(NC_FILE_INFO_T* h5)
2423 {
2424  NC_HDF5_FILE_INFO_T *hdf5_info;
2425 
2426  assert(h5 && h5->format_file_info);
2427  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2428 
2429  fprintf(stderr,"===== begin showopenobjects =====\n");
2430  reportopenobjects(0,hdf5_info->hdfid);
2431  fprintf(stderr,"===== end showopenobjects =====\n");
2432  fflush(stderr);
2433 }
2434 
2443 void
2444 showopenobjects(int ncid)
2445 {
2446  NC_FILE_INFO_T* h5 = NULL;
2447 
2448  /* Find our metadata for this file. */
2449  if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2450  fprintf(stderr,"failed\n");
2451  else
2452  showopenobjects5(h5);
2453  fflush(stderr);
2454 }
2455 
2467 int
2468 NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2469 {
2470  if(H5get_libversion(major,minor,release) < 0)
2471  return NC_EHDFERR;
2472  return NC_NOERR;
2473 }
2474 
2485 int
2486 NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2487 {
2488  NC_HDF5_FILE_INFO_T *hdf5_info;
2489  int stat = NC_NOERR;
2490  unsigned super;
2491  hid_t plist = -1;
2492 
2493  assert(h5 && h5->format_file_info);
2494  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2495 
2496  if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2497  {stat = NC_EHDFERR; goto done;}
2498  if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2499  {stat = NC_EHDFERR; goto done;}
2500  if(idp) *idp = (int)super;
2501 done:
2502  if(plist >= 0) H5Pclose(plist);
2503  return stat;
2504 }
2505 
2506 static int NC4_get_strict_att(NC_FILE_INFO_T*);
2507 static int NC4_walk(hid_t, int*);
2508 
2533 int
2534 NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2535 {
2536  int stat;
2537  int isnc4 = 0;
2538  int count;
2539 
2540  /* Look for NC3_STRICT_ATT_NAME */
2541  isnc4 = NC4_get_strict_att(h5);
2542  if(isnc4 > 0)
2543  goto done;
2544  /* attribute did not exist */
2545  /* => last resort: walk the HDF5 file looking for markers */
2546  count = 0;
2547  stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2548  &count);
2549  if(stat != NC_NOERR)
2550  isnc4 = 0;
2551  else /* Threshold is at least two matches */
2552  isnc4 = (count >= 2);
2553 
2554 done:
2555  return isnc4;
2556 }
2557 
2566 static int
2567 NC4_get_strict_att(NC_FILE_INFO_T *h5)
2568 {
2569  hid_t grpid = -1;
2570  hid_t attid = -1;
2571 
2572  /* Get root group ID. */
2573  grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2574 
2575  /* Try to extract the NC3_STRICT_ATT_NAME attribute */
2576  attid = H5Aopen_name(grpid, NC3_STRICT_ATT_NAME);
2577  H5Aclose(attid);
2578  return attid;
2579 }
2580 
2590 static int
2591 NC4_walk(hid_t gid, int* countp)
2592 {
2593  int ncstat = NC_NOERR;
2594  int i,j,na;
2595  ssize_t len;
2596  hsize_t nobj;
2597  herr_t err;
2598  int otype;
2599  hid_t grpid, dsid;
2600  char name[NC_HDF5_MAX_NAME];
2601 
2602  /* walk group members of interest */
2603  err = H5Gget_num_objs(gid, &nobj);
2604  if(err < 0) return err;
2605 
2606  for(i = 0; i < nobj; i++) {
2607  /* Get name & kind of object in the group */
2608  len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2609  if(len < 0) return len;
2610 
2611  otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2612  switch(otype) {
2613  case H5G_GROUP:
2614  grpid = H5Gopen(gid,name);
2615  NC4_walk(grpid,countp);
2616  H5Gclose(grpid);
2617  break;
2618  case H5G_DATASET: /* variables */
2619  /* Check for phony_dim */
2620  if(strcmp(name,"phony_dim")==0)
2621  *countp = *countp + 1;
2622  dsid = H5Dopen(gid,name);
2623  na = H5Aget_num_attrs(dsid);
2624  for(j = 0; j < na; j++) {
2625  hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2626  if(aid >= 0) {
2627  const NC_reservedatt* ra;
2628  ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2629  if(len < 0) return len;
2630  /* Is this a netcdf-4 marker attribute */
2631  /* Is this a netcdf-4 marker attribute */
2632  ra = NC_findreserved(name);
2633  if(ra != NULL)
2634  *countp = *countp + 1;
2635  }
2636  H5Aclose(aid);
2637  }
2638  H5Dclose(dsid);
2639  break;
2640  default:/* ignore */
2641  break;
2642  }
2643  }
2644  return ncstat;
2645 }
2646 
2647 
2648 /**************************************************/
2649 /* Filter registration support */
2650 
2651 static int
2652 filterlookup(int id)
2653 {
2654  int i;
2655  if(filters == NULL)
2656  filters = nclistnew();
2657  for(i=0;i<nclistlength(filters);i++) {
2658  NC_FILTER_INFO* x = nclistget(filters,i);
2659  if(x != NULL && x->id == id) return i; /* return position */
2660  }
2661  return -1;
2662 }
2663 
2664 static void
2665 reclaiminfo(NC_FILTER_INFO* info)
2666 {
2667  if(info != NULL)
2668  nullfree(info->info);
2669  nullfree(info);
2670 }
2671 
2672 static int
2673 filterremove(int pos)
2674 {
2675  NC_FILTER_INFO* info = NULL;
2676  if(filters == NULL)
2677  filters = nclistnew();
2678  if(pos < 0 || pos >= nclistlength(filters))
2679  return NC_EINVAL;
2680  info = nclistget(filters,pos);
2681  reclaiminfo(info);
2682  nclistremove(filters,pos);
2683  return NC_NOERR;
2684 }
2685 
2686 static NC_FILTER_INFO*
2687 dupfilterinfo(NC_FILTER_INFO* info)
2688 {
2689  NC_FILTER_INFO* dup = NULL;
2690  if(info == NULL) goto fail;
2691  if(info->info == NULL) goto fail;
2692  if((dup = calloc(1,sizeof(NC_FILTER_INFO))) == NULL) goto fail;
2693  *dup = *info;
2694  if((dup->info = calloc(1,sizeof(H5Z_class2_t))) == NULL) goto fail;
2695  {
2696  H5Z_class2_t* h5dup = (H5Z_class2_t*)dup->info;
2697  H5Z_class2_t* h5info = (H5Z_class2_t*)info->info;
2698  *h5dup = *h5info;
2699  }
2700  return dup;
2701 fail:
2702  reclaiminfo(dup);
2703  return NULL;
2704 }
2705 
2706 int
2707 nc4_filter_action(int op, int format, int id, NC_FILTER_INFO* info)
2708 {
2709  int stat = NC_NOERR;
2710  H5Z_class2_t* h5filterinfo = NULL;
2711  herr_t herr;
2712  int pos = -1;
2713  NC_FILTER_INFO* dup = NULL;
2714 
2715  if(format != NC_FILTER_FORMAT_HDF5)
2716  {stat = NC_ENOTNC4; goto done;}
2717 
2718  switch (op) {
2719  case FILTER_REG: /* Ignore id argument */
2720  if(info == NULL || info->info == NULL)
2721  {stat = NC_EINVAL; goto done;}
2722  if(info->version != NC_FILTER_INFO_VERSION
2723  || info->format != NC_FILTER_FORMAT_HDF5)
2724  {stat = NC_ENOTNC4; goto done;}
2725  h5filterinfo = info->info;
2726  /* Another sanity check */
2727  if(info->id != h5filterinfo->id)
2728  {stat = NC_EINVAL; goto done;}
2729  /* See if this filter is already defined */
2730  if((pos = filterlookup(id)) >= 0)
2731  {stat = NC_ENAMEINUSE; goto done;} /* Already defined */
2732  if((herr = H5Zregister(h5filterinfo)) < 0)
2733  {stat = NC_EFILTER; goto done;}
2734  /* Save a copy of the passed in info */
2735  if((dup = dupfilterinfo(info)) == NULL)
2736  {stat = NC_ENOMEM; goto done;}
2737  nclistpush(filters,dup);
2738  break;
2739  case FILTER_UNREG:
2740  if(id <= 0)
2741  {stat = NC_ENOTNC4; goto done;}
2742  /* See if this filter is already defined */
2743  if((pos = filterlookup(id)) < 0)
2744  {stat = NC_EFILTER; goto done;} /* Not defined */
2745  if((herr = H5Zunregister(id)) < 0)
2746  {stat = NC_EFILTER; goto done;}
2747  if((stat=filterremove(pos))) goto done;
2748  break;
2749  case FILTER_INQ:
2750  if(id <= 0)
2751  {stat = NC_ENOTNC4; goto done;}
2752  /* Look up the id in our local table */
2753  if((pos = filterlookup(id)) < 0)
2754  {stat = NC_EFILTER; goto done;} /* Not defined */
2755  if(info != NULL) {
2756  *info = *((NC_FILTER_INFO*)nclistget(filters,pos));
2757  }
2758  break;
2759  default:
2760  {stat = NC_EINTERNAL; goto done;}
2761  }
2762 done:
2763  return stat;
2764 }
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:405
#define NC_ENOTNC4
Attempting netcdf-4 operation on netcdf-3 file.
Definition: netcdf.h:448
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:138
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:276
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:438
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
#define NC_EINTERNAL
NetCDF Library Internal Error.
Definition: netcdf.h:431
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45
#define NC_STRING
string
Definition: netcdf.h:47
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
#define H5Z_FILTER_SZIP
ID of HDF SZIP filter.
Definition: dvarinq.c:16
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
size_t len
Length of VL data (in base type units)
Definition: netcdf.h:688
#define NC_ENAMEINUSE
String match to name in use.
Definition: netcdf.h:364
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:286
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:444
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:442
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:470
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:367
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:443
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:335
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:287
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:275
void * p
Pointer to VL data.
Definition: netcdf.h:689
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Free memory in a VLEN object.
Definition: dvlen.c:41
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:445
This is the type of arrays of vlens.
Definition: netcdf.h:687
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:285
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:379
#define NC_EPERM
Write to read only.
Definition: netcdf.h:336
#define NC_NOERR
No Error.
Definition: netcdf.h:325
#define NC_ENUM
enum types
Definition: netcdf.h:55
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:248
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46

Return to the Main Unidata NetCDF page.
Generated on Fri Aug 30 2019 08:26:47 for NetCDF. NetCDF is a Unidata library.