1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325 | int
saf_read_topo_relation(SAF_ParMode pmode, /* The parallel mode. */
SAF_Rel *rel, /* The topology relation to be read. */
SAF_RelTarget *target, /* Relation targeting information. */
void **abuf, /* The returned data. See saf_declare_topo_relation(). */
void **bbuf /* The returned data. See saf_declare_topo_relation(). */
)
{
SAF_ENTER(saf_read_topo_relation, SAF_PRECONDITION_ERROR);
static SAF_RelTarget rt_zero; /* Default targeting */
ss_scope_t scope=SS_SCOPE_NULL; /* Scope containing REL */
int scope_size; /* Size of the communicator for `scope' */
hid_t ftype=-1, mtype=-1; /* File and memory datatypes */
size_t mtype_size; /* Size in bytes of the memory datatype */
double timer_start=0;
ss_set_t sub=SS_SET_NULL, sup=SS_SET_NULL, Dstitching_set=SS_SET_NULL, Dusing_set=SS_SET_NULL;
ss_cat_t storage_decomp=SS_CAT_NULL, Dstitching_cat=SS_CAT_NULL, Dusing_cat=SS_CAT_NULL;
ss_collection_t storage_coll=SS_COLLECTION_NULL;
ss_relrep_t relrep=SS_RELREP_NULL, thisRelRep=SS_RELREP_NULL;
ss_blob_t rblob=SS_BLOB_NULL; /* range blob */
ss_blob_t dblob=SS_BLOB_NULL; /* domain blob */
hsize_t d_nelmts, r_nelmts; /* number of elements in domain and range blobs */
ss_cat_t sub_cat=SS_CAT_NULL;
ss_cat_t sup_cat=SS_CAT_NULL;
ss_collection_t sub_coll=SS_COLLECTION_NULL;
hbool_t desireHandles;
size_t bufferSize, IAcount, Acount, *Abuffer=NULL, *Bbuffer=NULL, Bcount, Nstitched, mapped_u;
size_t SS_stitching_Acount, NusingPerStitched, mapped_s;
size_t d, i, j, k, s, u;
int tmp_i, numberOfSubsetRelations, jj, *p, start, count, stride;
SAF_RelRep theRep, subsetRelationType;
SAF_Rel *IAbuf=NULL, *subsetRelationList=NULL;
hid_t IAtype=-1, Atype=-1, Btype=-1, SS_stitching_Atype=-1, SS_using_Atype=-1;
ss_file_t db=SS_FILE_NULL; /* File to which REL belongs and to which searches are limited. */
void *SS_stitching_Abuf=NULL, *SS_using_Abuf=NULL, *Abuf=NULL, *Bbuf=NULL;
SAF_REQUIRE(_saf_valid_pmode(pmode), SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
_saf_errmsg("PMODE must be valid"));
if (!_saf_is_participating_proc(pmode)) SAF_RETURN(-1);
ss_pers_scope((ss_pers_t*)rel, &scope);
ss_scope_comm(&scope, NULL, NULL, &scope_size);
if (!target) target = &rt_zero;
SAF_REQUIRE(SS_REL(rel), SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
_saf_errmsg("REL must be a valid relation handle"));
SAF_REQUIRE(abuf, SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
_saf_errmsg("ABUF must be non-null"));
SAF_REQUIRE(!bbuf || !SAF_XOR(*abuf,*bbuf), SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
_saf_errmsg("either both ABUF and BBUF point to NULL or both ABUF and BBUF do not point to NULL"));
SAF_REQUIRE((target->is_set && (SS_PERS_ISNULL(&target->decomp) || pmode==SAF_ALL || 1==scope_size)) || !target->is_set,
SAF_LOW_CHK_COST, SAF_PRECONDITION_ERROR,
_saf_errmsg("if targeting of storage decomposition is used, the read must be a SAF_ALL mode read "
"or the database must be opened on only a single processor"));
if (_SAF_GLOBALS.p.TraceTimes)
timer_start = _saf_wall_clock(false);
/* Cache some things for convenience */
ss_pers_file((ss_pers_t*)rel, &db);
sub = SS_REL(rel)->sub;
sup = SS_REL(rel)->sup;
sub_cat = SS_REL(rel)->sub_cat;
sup_cat = SS_REL(rel)->sup_cat;
storage_decomp = SS_REL(rel)->sub_decomp_cat;
relrep = SS_REL(rel)->rep_type;
rblob = SS_REL(rel)->r_blob;
dblob = SS_REL(rel)->d_blob;
/* Reading a topological relation that is stored on self is quite a bit different that one that is stored on a
* decomposition... */
if (_saf_is_self_decomp(&storage_decomp)) {
/* In this case the topological relation is stored on self. The number of items stored at A- and B-buffer data depends
* in the nature of the relation: structured, unstructured, or arbitrary. First of all structured topology has no
* buffers to read at all. */
if (SAF_STRUCTURED_ID==SS_RELREP(&relrep)->id)
goto theExit;
/* Range blob */
ss_blob_bound_f1(&rblob, NULL, NULL, &r_nelmts, &ftype);
mtype = H5Tget_native_type(ftype, H5T_DIR_DEFAULT);
mtype_size = H5Tget_size(mtype);
H5Tclose(ftype); ftype=-1;
/* As previosly noted, depending if the topological relation is unstructured or arbitrary we'll be needing either 1 or
* 2 buffers to receive the data. Which we then fill with data of the datatype found above. */
switch (SS_RELREP(&relrep)->id) {
case SAF_UNSTRUCTURED_ID:
/* Allocate space if necessary */
if (!*abuf && NULL==(*abuf=malloc(mtype_size)))
SAF_ERROR(SAF_MEMORY_ERROR,_saf_errmsg("unable to allocate space to read topological relation"));
if (bbuf && !*bbuf && r_nelmts && NULL==(*bbuf=malloc((size_t)r_nelmts*mtype_size)))
SAF_ERROR(SAF_MEMORY_ERROR,_saf_errmsg("unable to allocate space to read topological relation"));
/* Read the range (bbuf) data */
if (bbuf) {
if (ss_blob_bind_m1(&rblob, *bbuf, mtype, r_nelmts)<0 ||
NULL==ss_blob_read1(&rblob, (hsize_t)0, r_nelmts, SS_BLOB_UNBIND, NULL))
SAF_ERROR(SAF_FILE_ERROR,_saf_errmsg("unable to read relation data"));
}
/* Compute the # of range refs per member of collection */
if (NULL==_saf_getCollection_set(&sub, &sub_cat, &sub_coll))
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("unable to obtain collection for subset"));
tmp_i = SS_COLLECTION(&sub_coll)->count ? r_nelmts / SS_COLLECTION(&sub_coll)->count : 0;
if (NULL==_saf_convert(H5T_NATIVE_INT, &tmp_i, mtype, *abuf))
SAF_ERROR(SAF_FILE_ERROR,_saf_errmsg("data type is not appropriate for a relation"));
break;
case SAF_ARBITRARY_ID:
/* In this case, we need to the domain of the relation too */
ss_blob_bound_f1(&dblob, NULL, NULL, &d_nelmts, NULL);
/* Allocate space if necessary */
if (!*abuf && NULL==(*abuf=malloc((size_t)d_nelmts*mtype_size)))
SAF_ERROR(SAF_MEMORY_ERROR,_saf_errmsg("unable to allocate space to read topological relation"));
if (bbuf && !*bbuf && NULL==(*bbuf=malloc((size_t)r_nelmts*mtype_size)))
SAF_ERROR(SAF_MEMORY_ERROR,_saf_errmsg("unable to allocate space to read topological relation"));
/* Read the range data */
if (bbuf) {
if (ss_blob_bind_m1(&rblob, *bbuf, mtype, r_nelmts)<0 ||
NULL==ss_blob_read1(&rblob, (hsize_t)0, r_nelmts, SS_BLOB_UNBIND, NULL))
SAF_ERROR(SAF_FILE_ERROR,_saf_errmsg("unable to read relation data"));
}
/* Read the domain data */
if (ss_blob_bind_m1(&dblob, *abuf, mtype, d_nelmts)<0 ||
NULL==ss_blob_read1(&dblob, (hsize_t)0, d_nelmts, SS_BLOB_UNBIND, NULL))
SAF_ERROR(SAF_FILE_ERROR,_saf_errmsg("unable to read relation data"));
break;
default:
SAF_ERROR(SAF_FILE_ERROR,
_saf_errmsg("unknown topology relation type %s (%d)",
ss_string_ptr(SS_RELREP_P(&relrep,name)), SS_RELREP(&relrep)->id));
}
} else {
/* The relation is stored on a decomposition. There are two cases here: the caller wishes to receive the map on self
* decomposition (remapping the values to "global") or the caller wishes to recieve the (indirect) handles to the
* relations which actually have the data. The caller informs SAF of which of these two cases is desired by using the
* saf_target_topo_relation function. */
if (SS_PERS_ISNULL(&target->decomp)) {
desireHandles = TRUE;
} else if (_saf_is_self_decomp(&target->decomp)) {
desireHandles = FALSE;
} else {
desireHandles = TRUE;
}
if (desireHandles) {
/* Case 1: the caller wishes to receive the indirect handles. This is the common case: the caller should expect a
* single buffer (the A-buffer) of handles, we'll construct a buffer to hold the row numbers picked up from the
* blob... */
if (NULL==_saf_getCollection_set(&sup, &storage_decomp, &storage_coll))
SAF_ERROR(SAF_CONSTRAINT_ERROR,
_saf_errmsg("collection \"%s\" not found on set \"%s\"",
ss_string_ptr(SS_CAT_P(&storage_decomp,name)),
ss_string_ptr(SS_SET_P(&sup,name))));
bufferSize = SS_COLLECTION(&storage_coll)->count;
/* Read the relation links */
if (NULL==(*abuf=ss_array_get(SS_REL_P(rel,indirect_rels), ss_pers_tm, (size_t)0, bufferSize, *abuf)))
SAF_ERROR(SAF_MEMORY_ERROR,_saf_errmsg("unable to obtain topological relation data"));
} else {
/* Case 2: the caller wishes to receive the maps as though they had been stored on self rather than on a
* decomposition. The caller should expect the appropriate number of buffers, the number of items stored at A- and
* B-buffer data depends in the nature of the relation: structured, unstructured, or arbitrary. */
/* First we'll get the indirect handles using a recursive call. */
saf_get_count_and_type_for_topo_relation(pmode, rel, target, &theRep, &Acount, &Atype, &Bcount, &Btype);
switch (SS_RELREP(&theRep)->id) {
case SAF_ARBITRARY_ID:
SAF_ERROR(SAF_NOTIMPL_ERROR, _saf_errmsg("remapping of arbitrary topology not implemented yet"));
case SAF_STRUCTURED_ID:
SAF_ERROR(SAF_NOTIMPL_ERROR, _saf_errmsg("remapping of structured topology not implemented yet"));
case SAF_UNSTRUCTURED_ID:
Abuffer = malloc(sizeof(*Abuffer));
Bbuffer = malloc(Bcount*sizeof(*Bbuffer));
if (!Abuffer || !Bbuffer)
SAF_ERROR(SAF_MEMORY_ERROR, _saf_errmsg("out of memory"));
Abuffer[0] = SS_NOSIZE;
for (i=0; i<Bcount; i++) Bbuffer[i] = SS_NOSIZE;
saf_get_count_and_type_for_topo_relation(pmode, rel, NULL, NULL, &IAcount, &IAtype, NULL, NULL);
if (IAcount<1)
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("no indirect topo handles"));
if (saf_read_topo_relation(pmode, rel, NULL, (void**)&IAbuf, NULL)!=SAF_SUCCESS || !IAbuf)
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't retrieve indirect topo handles"));
for (d=0; d<IAcount; ++d) {
SS_stitching_Abuf = SS_using_Abuf = Abuf = Bbuf = NULL;
saf_describe_topo_relation(pmode, IAbuf+d, &Dstitching_set, &Dstitching_cat, &Dusing_set, &Dusing_cat,
NULL, NULL, NULL);
if (!SAF_EQUIV(&sub_cat,&Dstitching_cat))
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("remaping of boundary relations not supported yet"));
if (!SAF_EQUIV(&sup_cat,&Dusing_cat))
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("remaping of boundary relations not supported yet"));
if (!SAF_EQUIV(&sub,&Dstitching_set)) {
subsetRelationList=NULL;
saf_find_subset_relations(pmode, &db, &sub, &Dstitching_set, &sub_cat, &Dstitching_cat,
SAF_BOUNDARY_FALSE, SAF_BOUNDARY_FALSE, &numberOfSubsetRelations,
&subsetRelationList);
saf_describe_subset_relation(pmode, subsetRelationList+0, NULL, NULL, NULL, NULL, NULL, NULL,
&subsetRelationType, NULL);
if (numberOfSubsetRelations!=1 || !subsetRelationList)
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't retrieve subset relations"));
saf_get_count_and_type_for_subset_relation(pmode, subsetRelationList+0, NULL, &SS_stitching_Acount,
&SS_stitching_Atype, NULL, NULL);
saf_read_subset_relation(pmode, subsetRelationList+0, NULL, &SS_stitching_Abuf, NULL);
if (SAF_HSLAB_ID==SS_RELREP(&subsetRelationType)->id) {
if (SS_stitching_Acount!=3)
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid hyper slab relation"));
if (H5Tequal(SS_stitching_Atype,H5T_NATIVE_INT)) {
start = ((int *)SS_stitching_Abuf)[0];
count = ((int *)SS_stitching_Abuf)[1];
stride = ((int *)SS_stitching_Abuf)[2];
SS_stitching_Abuf = SS_FREE(SS_stitching_Abuf);
p = malloc(count*sizeof(int));
for (jj=0; jj<count; jj++)
p[jj] = start + jj * stride;
SS_stitching_Acount = count;
SS_stitching_Abuf = p;
} else {
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("invalid hyper slab type"));
}
}
subsetRelationList = SS_FREE(subsetRelationList);
}
if (!SAF_EQUIV(&sup, &Dusing_set)) {
subsetRelationList=NULL;
saf_find_subset_relations(pmode, &db, &sup, &Dusing_set, &sup_cat, &Dusing_cat, SAF_BOUNDARY_FALSE,
SAF_BOUNDARY_FALSE, &numberOfSubsetRelations, &subsetRelationList);
if (numberOfSubsetRelations!=1 || !subsetRelationList)
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't retrieve subset relations"));
saf_get_count_and_type_for_subset_relation(pmode, subsetRelationList+0, NULL, NULL, &SS_using_Atype,
NULL, NULL);
saf_read_subset_relation(pmode, subsetRelationList+0, NULL, &SS_using_Abuf, NULL);
subsetRelationList = SS_FREE(subsetRelationList);
}
saf_get_count_and_type_for_topo_relation(pmode, IAbuf+d, NULL, &thisRelRep, &Acount, &Atype, &Bcount, &Btype);
if (Acount > 0 && Bcount > 0) {
switch (SS_RELREP(&thisRelRep)->id) {
case SAF_ARBITRARY_ID:
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't remap arbitrary topo relation"));
case SAF_STRUCTURED_ID:
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't remap structured topo relation"));
case SAF_UNSTRUCTURED_ID:
saf_read_topo_relation(pmode, IAbuf+d, NULL, &Abuf, &Bbuf);
/* At this point we have every thing we need to add the contribution of this "domain" to the global
* map: one or more subset maps which show how a local member of a collection (such as a node or a
* zone), and the "local" topo maps. And all we must do is the actual composition and insertion
* into the map being assembled. */
/* Retrieve the number of "usings" per "stitched" (Abuf[0]). In a typical case "using" is "nodes"
* and "stitched" is "zones" (but it is not limited to this)... */
_saf_convert(Atype, Abuf, H5T_NATIVE_SIZE, &NusingPerStitched);
if (SS_NOSIZE==Abuffer[0]) {
Abuffer[0] = NusingPerStitched;
} else if (Abuffer[0]!=NusingPerStitched) {
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("inconsistant mapping factor"));
}
Nstitched = Bcount / NusingPerStitched;
/* Now we'll run through each of the members being "stitched" and for each of these we'll run
* though the N relations associated where N is number of "usings" per "stitched" (ie. nodes per
* zone). */
for (s=0; s<Nstitched; s++) {
if (SS_stitching_Abuf) {
_saf_convert(SS_stitching_Atype, (char*)SS_stitching_Abuf + s*H5Tget_size(SS_stitching_Atype),
H5T_NATIVE_SIZE, &mapped_s);
} else {
mapped_s = s;
}
for (i=0; i<NusingPerStitched; i++) {
/* Within each "stitched" (ie. zone) we'll run through its list of "usings" (ie. nodes). */
j = s * NusingPerStitched + i;
_saf_convert(Btype, (char*)Bbuf + j*H5Tget_size(Btype), H5T_NATIVE_SIZE, &u);
if (SS_using_Abuf) {
_saf_convert(SS_using_Atype, (char*)SS_using_Abuf + u*H5Tget_size(SS_using_Atype),
H5T_NATIVE_SIZE, &mapped_u);
} else {
mapped_u = u;
}
k = mapped_s * NusingPerStitched + i;
Bbuffer[k] = mapped_u;
}
}
break;
default:
SAF_ERROR(SAF_FILE_ERROR, _saf_errmsg("can't remap topo relation of unsupported type"));
}
Abuf = SS_FREE(Abuf);
Bbuf = SS_FREE(Bbuf);
SS_stitching_Abuf = SS_FREE(SS_stitching_Abuf);
SS_using_Abuf = SS_FREE(SS_using_Abuf);
}
}
if (abuf) {
*abuf = Abuffer;
Abuffer = NULL;
}
if (bbuf) {
*bbuf = Bbuffer;
Bbuffer = NULL;
}
break;
default:
SAF_ERROR(SAF_FILE_ERROR,_saf_errmsg("unsupported relation representation type"));
}
Abuffer = SS_FREE(IAbuf);
Abuffer = SS_FREE(Abuffer);
Bbuffer = SS_FREE(Bbuffer);
}
}
theExit:
if (_SAF_GLOBALS.p.TraceTimes)
_SAF_GLOBALS.CummReadTime += (_saf_wall_clock(false) - timer_start);
SAF_LEAVE(SAF_SUCCESS);
}
|