1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
 int
 saf_read_field(SAF_ParMode pmode,       /* The parallel mode. */
                SAF_Field *field,        /* The field which is to be read. */
                SAF_FieldTarget *target, /* Field targeting information. */
                int member_count,        /* A count of the number of members of the collection in which the field's dofs are n:1
                                          * associated with that are actually being written in this call. This value is ignored
                                          * if you are reading the entire field's dofs in this call (i.e., REQ_TYPE =
                                          * SAF_TOTALITY). Also note that as a convenience, we provide the macro
                                          * SAF_WHOLE_FIELD which expands to a comma separated list of appropriate values for
                                          * this argument and the next two, for the case in which the whole field is being read
                                          * in this call. */
                SAF_RelRep *req_type,    /* The type of I/O request. We use a relation representation type here to specify the
                                          * type of the partial request because it captures the necessary information. Pass
                                          * SAF_HSLAB if you are reading the dofs of a partial hyperslab of the members of the
                                          * associated collection. In this case, MEMBER_IDS points to 3 N-tuples of starts,
                                          * counts and strides of the hyperslab (hypersample) request. Pass SAF_TUPLES, if you
                                          * are reading the dofs for an arbitrary list of members of the associated collection.
                                          * In this case, the MEMBER_IDS points to a list of N-tuples. In both cases, 'N' is
                                          * the number of indexing dimensions in the associated collection. Finally, pass
                                          * SAF_TOTALITY if you are reading the entire field's set of dofs. */
                int *member_ids,         /* Depending on the value of REQ_TYPE, this argument points to 3 N-tuples storing,
                                          * respectively, the starts, counts and strides *in*each*dimension* of the associated
                                          * collection or to a list of MEMBER_COUNT N-tuples, each one identifying a single
                                          * member of the associated collection or to NULL in the case of a SAF_TOTALITY request. */
                void **Pbuf              /* [IN|OUT] A pointer to a buffer pointer which is to receive the values read.  The
                                          * caller may supply a pointer to a value of NULL if this function is to allocate a
                                          * buffer.  If the caller supplies a pointer to a non-NULL pointer (to a buffer) then
                                          * it is up to the caller to ensure that the buffer is of sufficient size to hold all
                                          * of the data retrieved.  The caller should use saf_describe_field() or
                                          * saf_get_count_and_type_for_field() to determine the datatype of the values read. */
                )
 {
     SAF_ENTER(saf_read_field, SAF_PRECONDITION_ERROR);
     double              timer_start=0;                  /* Start time for accumulating total field read times */
     ss_scope_t          scope=SS_SCOPE_NULL;            /* Scope containing FIELD */
     int                 scope_size;                     /* Size of the cummunicator for `scope' */
     static SAF_FieldTarget ft_zero;                     /* Default field targeting */
     hbool_t             has_been_written;               /* True if data has been written to the field */
     int                 retval;                         /* Return value for this function */
     ss_fieldtmpl_t      ftmpl=SS_FIELDTMPL_NULL;        /* Cached field template link from FIELD */
     ss_algebraic_t      algebraic=SS_ALGEBRAIC_NULL;    /* Cached algebraic type link from FIELD */
     ss_blob_t           dof_blob=SS_BLOB_NULL;          /* Cashed DOF blob from FIELD */
     hbool_t             desireHandles;                  /* Should we read field handles instead of dofs? */
     size_t              size;                           /* Number of elements to read */
     hsize_t             hsize;                          /* Size to bass to blob functions */
     size_t              offset;                         /* Index of first element to read */
     hid_t               memDatatype=-1;                 /* Type of data to store in returned PBUF array */
     hid_t               fileDatatype=-1;                /* Type of data stored in the file */
     SAF_Db              base_space_db;                  /* The database holding the base space of FIELD */
     size_t              numberOfGlobalDOFs;
     hid_t               theGlobalDOFType;
     SAF_FieldTmpl       theGlobalTemplate;
     SAF_Interleave      theGlobalComponentInterleave;
     size_t              theGlobalDOFSize;
     void                *theGlobalBuffer=NULL;
     char                *theGlobalPointer;
     size_t              numberOfLocalFields;
     int                 theGlobalComponentCount;
     SAF_Set             theGlobalSet;
     void                *buffer;
     SAF_Field           *theLocalFields;
     size_t              f;
     hid_t               theHandleType;
     size_t              numberOfGlobalValues;

     if (_SAF_GLOBALS.p.TraceTimes)
         timer_start = _saf_wall_clock(false);

     SAF_REQUIRE(_saf_valid_pmode(pmode), SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
                 _saf_errmsg("PMODE must be valid"));
     if (!_saf_is_participating_proc(pmode)) SAF_RETURN(-1);
     ss_pers_scope((ss_pers_t*)field, &scope);
     ss_scope_comm(&scope, NULL, NULL, &scope_size);
     if (!target) target = &ft_zero;

     SAF_REQUIRE(SS_FIELD(field), SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
                 _saf_errmsg("FIELD must be a valid field handle"));
     SAF_REQUIRE(Pbuf, SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
                 _saf_errmsg("PBUF must be non-null"));
     SAF_REQUIRE(_saf_is_valid_io_request(pmode, field, member_count, req_type, member_ids, 1),
                 SAF_HIGH_CHK_COST, SAF_PRECONDITION_ERROR,
                 _saf_errmsg("if partial I/O request, associated collection must be 1D indexed, REQ_TYPE must be SAF_HSLAB "
                             "or a single SAF_TUPLE and field's interleave, if multi-component, must be SAF_INTERLEAVE_VECTOR"));
     SAF_REQUIRE(SS_PERS_ISNULL(&target->decomp) || pmode==SAF_ALL || 1==scope_size, SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
                 _saf_errmsg("if field targeting of storage decomposition is used, the read must be a SAF_ALL mode read or the "
                             "database must be opened on only a single processor"));

     /* If there is no data written to this field, try using _saf_read_comp_field instead. */
     saf_data_has_been_written_to_field(pmode, field, &has_been_written);
     if(!has_been_written) {
         retval = _saf_read_comp_field(pmode, field, member_count, req_type, member_ids, Pbuf);
         SAF_RETURN(retval);
     }

     /* Cache some stuff for convenience */
     ftmpl = SS_FIELD(field)->ftmpl;
     algebraic = SS_FIELDTMPL(&ftmpl)->algebraic;
     dof_blob = SS_FIELD(field)->dof_blob;

     /* We can't get here without passing the valid_io_request pre-condition and all the limitations it currently
      * imposes. So, we know member_ids is either an array of 3 ints {start, count, stride} where stride is constrained
      * to 1 for SAF_HSLAB or an array of 1 int {index} for SAF_TUPLES. Regardless, member_ids[0] is the starting position
      * and member_count is the size of the request. */
     if (_saf_is_self_decomp(SS_FIELD_P(field,storage_decomp_cat)) && !SS_ALGEBRAIC(&algebraic)->indirect) {
         /* The stored values in the field are real field DOF values. */
         ss_blob_bound_f1(&dof_blob, NULL, NULL, &hsize, &fileDatatype);
         if (SAF_TOTALITY_ID==SS_RELREP(req_type)->id) {
             size = (size_t)hsize;
             offset = 0;
         } else {
             offset = member_ids[0];
             size = member_count;
         }
         memDatatype = H5Tget_native_type(target->data_type>0?target->data_type:fileDatatype, H5T_DIR_DEFAULT);

         /*  Allocate a return buffer if the caller did not provide one... */
         if (!*Pbuf && NULL==(*Pbuf=malloc(size*H5Tget_size(memDatatype))))
             SAF_ERROR(SAF_MEMORY_ERROR, _saf_errmsg("unable to allocate space to read field"));

         /* Read field DOFs from blob dataset filling the return buffer... */
         if (ss_blob_bind_m1(&dof_blob, *Pbuf, memDatatype, (hsize_t)size)<0 ||
             NULL==ss_blob_read1(&dof_blob, (hsize_t)offset, (hsize_t)size, SS_BLOB_UNBIND, NULL))
             SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("unable to read field dofs"));
     } else {
         /* The field is stored on a decomposition.  The stored values are handles to the fields on the parts forming the
          * decomposition.  The datatype is known to be handles (to fields) and the blob record tells how many. However the
          * caller may have used the saf_target_field() function to request the field to be remapped. */
         if (SAF_TOTALITY_ID==SS_RELREP(req_type)->id) {
             offset = 0;
             size = ss_array_nelmts(SS_FIELD_P(field,indirect_fields));
         } else {
             offset = member_ids[0];
             size = member_count;
         }
         if (SS_PERS_ISNULL(&target->decomp)) {
             desireHandles = TRUE;
         } else if (_saf_is_self_decomp(&target->decomp)) {
             desireHandles = FALSE;
         } else {
             desireHandles = TRUE;
         }
         if (desireHandles) {
             /* We need to adjust the offset for the case of a homogeneous field of fields (probably a state field) because all
              * the data is compressed into a single blob. For an inhomogeneous field the size and count need no adjustment. */
             if (_saf_is_self_decomp(SS_FIELD_P(field,storage_decomp_cat)) && SS_ALGEBRAIC(&algebraic)->indirect) {
                 assert(SS_FIELDTMPL(&ftmpl)->num_comps>=0);
                 size *= SS_FIELDTMPL(&ftmpl)->num_comps;
                 offset *= SS_FIELDTMPL(&ftmpl)->num_comps;
             }
             if (NULL==(*Pbuf = ss_array_get(SS_FIELD_P(field,indirect_fields), ss_pers_tm, offset, size, *Pbuf)))
                 SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("unable to read field links"));
         } else {
             /* The caller wishes to receive the DOFs as though they had been stored on self rather than on a
              * decomposition. What we need to do is to allocate a buffer big enough for the mapped field. We then read the each
              * local field one at a time and move the DOFs from the local field buffer to the global buffer. */

             /* ISSUE: For an indirect field the the local fields are all "similar". That is, they have the same algebraic type,
              *        association category, units and such. This function should check for this but doesn't. In the future
              *        some differences can be smoothed-over (such as units) but some probably can not (such as algebraic
              *        type). */
             /* ISSUE: The proper use of PMODE is not fully worked out. */
             /* ISSUE: Multiple indirection may actually fall out of this solution but that is not at all clear. */

             saf_get_count_and_type_for_field(pmode, field, target, &numberOfGlobalDOFs, &theGlobalDOFType);
             saf_describe_field(pmode, field, &theGlobalTemplate, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                                &theGlobalComponentCount, NULL, &theGlobalComponentInterleave, NULL);
             if (target->is_set) theGlobalComponentInterleave = target->comp_intlv;
 #if 0
             /*  This needs to be fixed.  The problem is if theGlobalComponentCount == -1,
              *  then we don't know if we need to check the interleave.  We will only
              *  find out once we have read in the first block if we have multiple components,
                  *  so the test needs to be moved, not sure where at this point. */
             if (theGlobalComponentCount > 1) {
                 if (theGlobalComponentInterleave != SAF_INTERLEAVE_VECTOR &&
                     theGlobalComponentInterleave != SAF_INTERLEAVE_COMPONENT) {
                     SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid component interleave"));
                 }
             }
 #endif

             /*  Allocate storage for the global buffer. */
             theGlobalDOFSize = H5Tget_size(theGlobalDOFType);
             if (*Pbuf) {
                 theGlobalBuffer = *Pbuf;
             } else {
                 if (numberOfGlobalDOFs == 0) {
                     SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("field has no global DOFs"));
                 }
                 theGlobalBuffer = malloc(numberOfGlobalDOFs * H5Tget_size(theGlobalDOFType));
                 if (theGlobalBuffer == NULL) {
                     SAF_ERROR(SAF_MEMORY_ERROR, _saf_errmsg("can't allocate global DOF buffer"));
                 }
             }
             theGlobalPointer = theGlobalBuffer;

             saf_get_count_and_type_for_field(pmode, field, NULL, &numberOfLocalFields, &theHandleType);
             if (numberOfLocalFields < 1) {
                 SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("no indirect field handles"));
             }
             saf_describe_field(pmode, field, &theGlobalTemplate, NULL, &theGlobalSet, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                                NULL, NULL, NULL, NULL, NULL);
             saf_describe_field_tmpl(pmode, &theGlobalTemplate, NULL, NULL, NULL, NULL, NULL, NULL);
             buffer = NULL;
             saf_read_field(pmode, field, NULL, SAF_WHOLE_FIELD, &buffer);
             if (buffer == NULL) {
                 SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't read field handles"));
             }
             theLocalFields = buffer;
             for (f=0; f<numberOfLocalFields; f++) {
                 size_t   numberOfLocalDOFs;
                 hid_t theLocalDOFType;

                 saf_get_count_and_type_for_field(pmode, theLocalFields+f, target, &numberOfLocalDOFs, &theLocalDOFType);
                 if (target->data_type>0)
                     theLocalDOFType = target->data_type;
                 if (0 < numberOfLocalDOFs) {
                     void          *Abuffer;
                     size_t         AbufSize;
                     hid_t          AbufType;
                     void          *Bbuffer;
                     size_t         BbufSize;
                     hid_t          BbufType;
                     int            C;
                     size_t         G,L;
                     size_t         CGstride, Gstride;
                     size_t         CLstride, Lstride;
                     int            numberOfSubsetRels;
                     SAF_Cat        theCat;
                     int            theLocalComponentCount;
                     SAF_Field     *theLocalComponentFields;
                     SAF_Interleave theLocalComponentInterleave;
                     size_t         numberOfLocalValues;
                     void          *theLocalBuffer;
                     size_t         theLocalDOFSize;
                     char          *theLocalPointer;
                     SAF_Set        theLocalSet;
                     SAF_FieldTmpl  theLocalTemplate;
                     SAF_Rel       *theSubsetRels;
                     SAF_IndexSpec  ispec;
                     size_t         origin;
                     SAF_RelRep     srtype;

                     theLocalComponentFields = NULL;
                     saf_describe_field(pmode, theLocalFields+f, &theLocalTemplate, NULL, &theLocalSet, NULL, NULL, NULL, &theCat,
                                        NULL, NULL, NULL, NULL, &theLocalComponentCount, &theLocalComponentFields, NULL, NULL);

                     /* If the component count is negative then it is an indirect field and the component count must be gotten
                      * by recursing on the indirect fields until a valid component count is encountered. */
                     if (theLocalComponentCount == -1)
                         _saf_numberOfComponentsOf_field(pmode, theLocalFields+f, &theLocalComponentCount);

                     numberOfGlobalValues = numberOfGlobalDOFs / theLocalComponentCount;
                     numberOfLocalValues = numberOfLocalDOFs / theLocalComponentCount;

                     if (theLocalComponentFields != NULL) {
                         int c;
                         char *thePointer;

                         theLocalDOFSize = H5Tget_size(theLocalDOFType);
                         theLocalBuffer  = malloc(numberOfLocalDOFs*theLocalDOFSize);
                         if (theLocalBuffer == NULL)
                             SAF_ERROR(SAF_MEMORY_ERROR, _saf_errmsg("can't allocate local DOF buffer"));

                         thePointer = theLocalBuffer;
                         for(c=0; c<theLocalComponentCount; c++) {
                             saf_read_field(pmode, theLocalComponentFields+c, NULL, SAF_WHOLE_FIELD, (void **)&thePointer);
                             thePointer += theLocalDOFSize * numberOfLocalValues;
                         }

                         free(theLocalComponentFields);

                         theLocalComponentInterleave = SAF_INTERLEAVE_COMPONENT;
                     } else {
                         theLocalDOFSize = H5Tget_size(theLocalDOFType);
                         theLocalBuffer  = malloc(numberOfLocalDOFs*theLocalDOFSize);
                         if (theLocalBuffer == NULL)
                             SAF_ERROR(SAF_MEMORY_ERROR, _saf_errmsg("can't allocate local DOF buffer"));
                         saf_read_field(pmode, theLocalFields+f, target, SAF_WHOLE_FIELD, &theLocalBuffer);
                         theLocalComponentInterleave = target->comp_intlv;
                     }
                     theLocalPointer = theLocalBuffer;
                     if (theLocalComponentCount > 1) {
                         if (theLocalComponentInterleave != SAF_INTERLEAVE_VECTOR &&
                             theLocalComponentInterleave != SAF_INTERLEAVE_COMPONENT) {
                             SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid component interleave"));
                         }
                     }
                     saf_describe_field_tmpl(pmode, &theLocalTemplate, NULL, NULL, NULL, NULL, NULL, NULL);
                     saf_describe_collection(pmode, &theLocalSet, &theCat, NULL, NULL, &ispec, NULL, NULL);
                     origin = ispec.origins[0];

                     numberOfSubsetRels = 0;
                     theSubsetRels      = NULL;
 #ifdef SSLIB_SUPPORT_PENDING /* Fix me in Phase-II */
                     /* ISSUE: When remapping an indirect field we only look in the top-scope of the file containing the
                      *        field's base space when searching for the subset relations. [rpm 2004-05-24] */
 #endif /*SSLIB_SUPPORT_PENDING*/
                     ss_pers_file((ss_pers_t*)&theGlobalSet, &base_space_db);
                     saf_find_subset_relations(pmode, &base_space_db, &theGlobalSet, &theLocalSet, &theCat, &theCat,
                                               SAF_BOUNDARY_FALSE, SAF_BOUNDARY_FALSE, &numberOfSubsetRels, &theSubsetRels);
                     if (numberOfSubsetRels!=1 || !theSubsetRels)
                         SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't locate subset relation for field stored on domain"));
                     saf_describe_subset_relation(pmode, theSubsetRels, NULL, NULL, NULL, NULL, NULL, NULL, &srtype, NULL);
                     saf_get_count_and_type_for_subset_relation(pmode, theSubsetRels, NULL, &AbufSize, &AbufType, &BbufSize,
                                                                &BbufType);
                     Abuffer = NULL;
                     Bbuffer = NULL;
                     saf_read_subset_relation(pmode, theSubsetRels, NULL, &Abuffer, &Bbuffer);
                     if (!Abuffer)
                         SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't read subset for domain"));
                     if (Bbuffer)
                         SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("unexpected B-buffer for subset for domain"));

                     /* Handle hyperslabs, by creating a tuple out of the hyperslab.  This isn't very efficient but it is the
                      * most economical to program. */
                     if (SAF_EQUIV(&srtype, SAF_HSLAB)) {
                         if (AbufSize != 3)
                             SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid hyper slab relation"));
                         if (H5Tequal(AbufType,H5T_NATIVE_INT)) {
                             int i;
                             int *p;
                             int start;
                             int count;
                             int stride;

                             start  = ((int *)Abuffer)[0];
                             count  = ((int *)Abuffer)[1];
                             stride = ((int *)Abuffer)[2];
                             free(Abuffer);

                             p = malloc(count*sizeof(int));
                             for (i=0; i<count; i++) {
                                 p[i] = start + i * stride;
                             }
                             AbufSize = count;
                             Abuffer = p;
                         }
                     }

                     if (AbufSize != numberOfLocalValues)
                         SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("subset size != number of local DOFs"));

                     if (theGlobalComponentInterleave == SAF_INTERLEAVE_VECTOR) {
                         CGstride = 1;
                         Gstride = theGlobalComponentCount;
                     } else {
                         CGstride = numberOfGlobalValues;
                         Gstride = 1;
                     }
                     if (theLocalComponentInterleave == SAF_INTERLEAVE_VECTOR) {
                         CLstride = 1;
                         Lstride = theLocalComponentCount;
                     } else {
                         CLstride = numberOfLocalValues;
                         Lstride = 1;
                     }

                     if (H5Tequal(AbufType,H5T_NATIVE_INT)) {
                         int *p = (int*)Abuffer;
                         void *buf = malloc(MAX(theLocalDOFSize, theGlobalDOFSize));
                         for (C=0; C<theLocalComponentCount; ++C) {
                             for (L=0; L<numberOfLocalValues; ++L) {
                                 G = (size_t)(p[L]) - origin;
                                 if (numberOfGlobalValues <= G) {
                                     SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid index found in A-buffer"));
                                 }
                                 memcpy(buf, theLocalPointer+(L*Lstride+C*CLstride)*theLocalDOFSize, theLocalDOFSize);
                                 H5Tconvert(theLocalDOFType, theGlobalDOFType, 1, buf, NULL, H5P_DEFAULT);
                                 memcpy(theGlobalPointer+(G*Gstride+C*CGstride)*theGlobalDOFSize, buf, theGlobalDOFSize);
                             }
                         }
                         SS_FREE(buf);
                     } else if (H5Tequal(AbufType,H5T_NATIVE_UINT)) {
                         unsigned int *p = (unsigned int *)Abuffer;
                         void *buf = malloc(MAX(theLocalDOFSize, theGlobalDOFSize));
                         for (C=0; C<theLocalComponentCount; ++C) {
                             for (L=0; L<numberOfLocalValues; ++L) {
                                 G = (size_t)(p[L]) - origin;
                                 if (numberOfGlobalValues <= G) {
                                     SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid index found in A-buffer"));
                                 }
                                 memcpy(buf, theLocalPointer+(L*Lstride+C*CLstride)*theLocalDOFSize, theLocalDOFSize);
                                 H5Tconvert(theLocalDOFType, theGlobalDOFType, 1, buf, NULL, H5P_DEFAULT);
                                 memcpy(theGlobalPointer+(G*Gstride+C*CGstride)*theGlobalDOFSize, buf, theGlobalDOFSize);
                             }
                         }
                         SS_FREE(buf);
                     } else if (H5Tequal(AbufType,H5T_NATIVE_LONG)) {
                         long *p = (long *)Abuffer;
                         void *buf = malloc(MAX(theLocalDOFSize, theGlobalDOFSize));
                         for (C=0; C<theLocalComponentCount; ++C) {
                             for (L=0; L<numberOfLocalValues; ++L) {
                                 G = (size_t)(p[L]) - origin;
                                 if (numberOfGlobalValues <= G) {
                                     SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid index found in A-buffer"));
                                 }
                                 memcpy(buf, theLocalPointer+(L*Lstride+C*CLstride)*theLocalDOFSize, theLocalDOFSize);
                                 H5Tconvert(theLocalDOFType, theGlobalDOFType, 1, buf, NULL, H5P_DEFAULT);
                                 memcpy(theGlobalPointer+(G*Gstride+C*CGstride)*theGlobalDOFSize, buf, theGlobalDOFSize);
                             }
                         }
                         SS_FREE(buf);
                     } else if (H5Tequal(AbufType,H5T_NATIVE_ULONG)) {
                         unsigned long *p = (unsigned long *)Abuffer;
                         void *buf = malloc(MAX(theLocalDOFSize, theGlobalDOFSize));
                         for (C=0; C<theLocalComponentCount; ++C) {
                             for (L=0; L<numberOfLocalValues; ++L) {
                                 G = (size_t)(p[L]) - origin;
                                 if (numberOfGlobalValues <= G) {
                                     SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid index found in A-buffer"));
                                 }
                                 memcpy(buf, theLocalPointer+(L*Lstride+C*CLstride)*theLocalDOFSize, theLocalDOFSize);
                                 H5Tconvert(theLocalDOFType, theGlobalDOFType, 1, buf, NULL, H5P_DEFAULT);
                                 memcpy(theGlobalPointer+(G*Gstride+C*CGstride)*theGlobalDOFSize, buf, theGlobalDOFSize);
                             }
                         }
                         SS_FREE(buf);
                     } else {
                         SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("unsupported type for subset for domain"));
                     }
                     free(Abuffer);
                     free(theSubsetRels);
                     free(theLocalBuffer);
                 }
             }
             free(buffer);
             if (*Pbuf == NULL)
                 *Pbuf = theGlobalBuffer;
         }
     }

     if (_SAF_GLOBALS.p.TraceTimes)
         _SAF_GLOBALS.CummReadTime += (_saf_wall_clock(false) - timer_start);

     SAF_LEAVE(SAF_SUCCESS);
 }