diff --git a/hdfview/src/main/java/hdf/view/TableView/DataDisplayConverterFactory.java b/hdfview/src/main/java/hdf/view/TableView/DataDisplayConverterFactory.java index 50651015..a338e085 100644 --- a/hdfview/src/main/java/hdf/view/TableView/DataDisplayConverterFactory.java +++ b/hdfview/src/main/java/hdf/view/TableView/DataDisplayConverterFactory.java @@ -73,15 +73,9 @@ public static HDFDisplayConverter getDataDisplayConverter(final DataFormat dataO dataFormatReference = dataObject; - Datatype dtype = dataObject.getDatatype(); - - // For VLEN(compound), use compound base type so CompoundDataDisplayConverter is created - if (dtype.isVLEN() && !dtype.isVarStr() && dtype.getDatatypeBase() != null && - dtype.getDatatypeBase().isCompound()) { - dtype = dtype.getDatatypeBase(); - } - - HDFDisplayConverter converter = getDataDisplayConverter(dtype); + // A top-level vlen-of-compound is a single column; dispatch on the dataset's own + // datatype so getDataDisplayConverter(VLEN) makes a VlenDataDisplayConverter. + HDFDisplayConverter converter = getDataDisplayConverter(dataObject.getDatatype()); return converter; } @@ -237,7 +231,7 @@ private static class CompoundDataDisplayConverter extends HDFDisplayConverter { CompoundDataFormat compoundFormat = (CompoundDataFormat)dataFormatReference; List localSelectedTypes = - DataFactoryUtils.filterNonSelectedMembers(compoundFormat, dtype); + DataFactoryUtils.filterNonSelectedMembers(compoundFormat, dtype, false); log.trace("setting up {} base HDFDisplayConverters", localSelectedTypes.size()); @@ -331,10 +325,16 @@ public Object canonicalToDisplayValue(Object value) Object curObject = cmpdList.get(i); if (curObject instanceof List) buffer.append(memberTypeConverters[i].canonicalToDisplayValue(curObject)); - else { + else if (curObject != null && curObject.getClass().isArray()) { + // Array-of-compound: the member is a column-array indexed by row. Object dataArrayValue = Array.get(curObject, cellRowIdx); buffer.append(memberTypeConverters[i].canonicalToDisplayValue(dataArrayValue)); } + else { + // A single compound element (e.g. one element of a vlen-of-compound): + // the member is already a scalar value, not a per-row column. + buffer.append(memberTypeConverters[i].canonicalToDisplayValue(curObject)); + } } buffer.append("}"); } @@ -610,7 +610,14 @@ public Object canonicalToDisplayValue(Object value) try { Object obj; Object convertedValue; - int arrLen = Array.getLength(value); + + // Column-expanded vlen cells hand us a single scalar; defer to base converter. + if (!value.getClass().isArray() && !(value instanceof List)) { + buffer.append(baseTypeConverter.canonicalToDisplayValue(value)); + return buffer; + } + + int arrLen = (value instanceof List) ? ((List)value).size() : Array.getLength(value); log.trace("canonicalToDisplayValue({}): array length={}", value, arrLen); @@ -621,7 +628,7 @@ public Object canonicalToDisplayValue(Object value) if (i > 0) buffer.append(", "); - obj = Array.get(value, i); + obj = (value instanceof List) ? ((List)value).get(i) : Array.get(value, i); convertedValue = baseTypeConverter.canonicalToDisplayValue(obj); diff --git a/hdfview/src/main/java/hdf/view/TableView/DataFactoryUtils.java b/hdfview/src/main/java/hdf/view/TableView/DataFactoryUtils.java index 197b6154..5c0bd12f 100644 --- a/hdfview/src/main/java/hdf/view/TableView/DataFactoryUtils.java +++ b/hdfview/src/main/java/hdf/view/TableView/DataFactoryUtils.java @@ -47,6 +47,40 @@ public class DataFactoryUtils { /** the CMPD_START_IDX_MAP_INDEX value. */ public static final int CMPD_START_IDX_MAP_INDEX = 1; + /** + * Number of flat leaf names a Datatype contributes to the flat-name list + * produced by H5Datatype.extractCompoundInfo. The rules: + * compound - sum of children's counts (no header entry) + * array(compound) - 1 header + sum of inner compound's children's counts + * array(atomic) - 1 + * vlen(any) - 1 (header only; vlen recursion is disabled) + * atomic, varstr - 1 + */ + public static int countLeafNames(Datatype t) + { + if (t == null) + return 1; + if (t.isCompound()) { + int sum = 0; + List children = t.getCompoundMemberTypes(); + if (children != null) + for (Datatype child : children) + sum += countLeafNames(child); + return sum; + } + if (t.isArray()) { + Datatype base = t.getDatatypeBase(); + if (base != null && base.isCompound()) { + int sum = 1; + for (Datatype child : base.getCompoundMemberTypes()) + sum += countLeafNames(child); + return sum; + } + return 1; + } + return 1; + } + /** * Given a CompoundDataFormat, as well as a compound datatype, removes the * non-selected datatypes from the List of datatypes inside the compound @@ -62,34 +96,35 @@ public class DataFactoryUtils { public static List filterNonSelectedMembers(CompoundDataFormat dataFormat, final Datatype compoundType) { + return filterNonSelectedMembers(dataFormat, compoundType, true); + } + + /** + * Variant of {@link #filterNonSelectedMembers(CompoundDataFormat, Datatype)} that + * skips the selected-member filter for inner (non-top-level) compounds. + * + * The dataset's flat selected-member list only enumerates top-level leaves; inner + * compound members can't be individually deselected. Filtering an inner compound + * against that list would spuriously remove every non-compound member. + */ + public static List filterNonSelectedMembers(CompoundDataFormat dataFormat, + final Datatype compoundType, boolean isTopLevel) + { + List selectedTypes = new ArrayList<>(compoundType.getCompoundMemberTypes()); + if (!isTopLevel) + return selectedTypes; + List allSelectedTypes = Arrays.asList(dataFormat.getSelectedMemberTypes()); if (allSelectedTypes == null) { log.debug("filterNonSelectedMembers(): selected compound member datatype list is null"); return null; } - /* - * Make sure to make a copy of the compound datatype's member list, as we will - * make modifications to the list when members aren't selected. - */ - List selectedTypes = new ArrayList<>(compoundType.getCompoundMemberTypes()); - - /* - * Among the datatypes within this compound type, only keep the ones that are - * actually selected in the dataset. - */ Iterator localIt = selectedTypes.iterator(); while (localIt.hasNext()) { Datatype curType = localIt.next(); - - /* - * Since the passed in allSelectedMembers list is a flattened out datatype - * structure, we want to leave the nested compound Datatypes inside our local - * list of datatypes. - */ if (curType.isCompound()) continue; - if (!allSelectedTypes.contains(curType)) localIt.remove(); } @@ -189,14 +224,19 @@ else if (base.isArray()) { } log.trace("buildColIdxToStartIdxMap(): arrSize after base={}", arrSize); } + // A vlen member is always a single column rendered by VlenDataProvider as the + // whole sequence (a brace-string, recursing for nested compounds). A top-level + // vlen-of-compound is already peeled to its inner compound at the dataset level, + // so any vlen reaching here is a member - never expanded into per-element columns. if (nestedCompoundType != null) { - List cmpdSelectedTypes = filterNonSelectedMembers(dataFormat, nestedCompoundType); + List cmpdSelectedTypes = + filterNonSelectedMembers(dataFormat, nestedCompoundType, false); /* - * For Array/Vlen of Compound types, we repeat the compound members n times, - * where n is the number of array elements of variable-length elements. - * Therefore, we repeat our mapping for these types n times. + * For Array of Compound types, we repeat the compound members n times, + * where n is the number of array elements. For a top-level + * vlen-of-compound, arrSize is 1 (one column per inner member). */ for (int j = 0; j < arrSize; j++) { buildColIdxToProviderMap(outMap, dataFormat, cmpdSelectedTypes, curMapIndex, @@ -204,11 +244,15 @@ else if (base.isArray()) { } } else if (curType.isCompound()) { - List cmpdSelectedTypes = filterNonSelectedMembers(dataFormat, curType); + List cmpdSelectedTypes = filterNonSelectedMembers(dataFormat, curType, false); buildColIdxToProviderMap(outMap, dataFormat, cmpdSelectedTypes, curMapIndex, curProviderIndex, depth + 1); } + else if (curType.isVLEN() && !curType.isVarStr()) { + for (int j = 0; j < arrSize; j++) + outMap.put(curMapIndex[0]++, curProviderIndex[0]); + } else outMap.put(curMapIndex[0]++, curProviderIndex[0]); @@ -297,9 +341,12 @@ else if (base.isArray()) { } log.trace("buildRelColIdxToStartIdxMap(): arrSize after base={}", arrSize); } + // Single column per vlen member (see buildColIdxToProviderMap). No per-element + // expansion; top-level vlen-of-compound is already peeled at the dataset level. if (nestedCompoundType != null) { - List cmpdSelectedTypes = filterNonSelectedMembers(dataFormat, nestedCompoundType); + List cmpdSelectedTypes = + filterNonSelectedMembers(dataFormat, nestedCompoundType, false); /* * For Array/Vlen of Compound types, we repeat the compound members n times, @@ -318,11 +365,21 @@ else if (curType.isCompound()) { if (depth == 0) curStartIdx[0] = curMapIndex[0]; - List cmpdSelectedTypes = filterNonSelectedMembers(dataFormat, curType); + List cmpdSelectedTypes = filterNonSelectedMembers(dataFormat, curType, false); buildRelColIdxToStartIdxMap(outMap, dataFormat, cmpdSelectedTypes, curMapIndex, curStartIdx, depth + 1); } + else if (curType.isVLEN() && !curType.isVarStr()) { + for (int j = 0; j < arrSize; j++) { + if (depth == 0) { + outMap.put(curMapIndex[0], curMapIndex[0]); + curMapIndex[0]++; + } + else + outMap.put(curMapIndex[0]++, curStartIdx[0]); + } + } else { if (depth == 0) { outMap.put(curMapIndex[0], curMapIndex[0]); @@ -333,4 +390,42 @@ else if (curType.isCompound()) { } } } + + /** + * Return true when the datatype tree contains a construct whose table-view + * write path is not symmetric with the (recently expanded) display path, + * so a single-cell edit cannot be reliably mapped back to storage. + */ + public static boolean isUnsafeForWrite(Datatype dtype) { return isUnsafe(dtype, false); } + + private static boolean isUnsafe(Datatype dtype, boolean insideCompound) + { + if (dtype == null) + return false; + + if (dtype.isVLEN() && !dtype.isVarStr()) + return true; + + if (dtype.isArray()) { + Datatype base = dtype.getDatatypeBase(); + if (base != null && (base.isCompound() || base.isArray() || (base.isVLEN() && !base.isVarStr()))) + return true; + return insideCompound; + } + + if (dtype.isCompound()) { + if (insideCompound) + return true; + List members = dtype.getCompoundMemberTypes(); + if (members != null) { + for (Datatype m : members) { + if (isUnsafe(m, true)) + return true; + } + } + return false; + } + + return false; + } } diff --git a/hdfview/src/main/java/hdf/view/TableView/DataProviderFactory.java b/hdfview/src/main/java/hdf/view/TableView/DataProviderFactory.java index 2d515da3..487a0495 100644 --- a/hdfview/src/main/java/hdf/view/TableView/DataProviderFactory.java +++ b/hdfview/src/main/java/hdf/view/TableView/DataProviderFactory.java @@ -74,17 +74,10 @@ public static HDFDataProvider getDataProvider(final DataFormat dataObject, final dataFormatReference = dataObject; - Datatype dtype = dataObject.getDatatype(); - - // For VLEN(compound), use the compound base type so CompoundDataProvider is created. - // The VLEN aspect is handled in the read path (H5DreadVL), and each member's data - // is a String[] of brace-enclosed values. - if (dtype.isVLEN() && !dtype.isVarStr() && dtype.getDatatypeBase() != null && - dtype.getDatatypeBase().isCompound()) { - dtype = dtype.getDatatypeBase(); - } - - HDFDataProvider dataProvider = getDataProvider(dtype, dataBuf, dataTransposed); + // A top-level vlen-of-compound is a single column showing the whole sequence, so + // dispatch on the dataset's own datatype: getDataProvider(VLEN) makes a + // VlenDataProvider (read path supplies one bare ArrayList[] of nested-List elements). + HDFDataProvider dataProvider = getDataProvider(dataObject.getDatatype(), dataBuf, dataTransposed); return dataProvider; } @@ -329,8 +322,11 @@ public Object getDataValue(Object obj, int index) try { if (obj instanceof ArrayList) theValue = ((ArrayList)obj).get(index); - else + else if (obj != null && obj.getClass().isArray()) theValue = Array.get(obj, index); + else + // Scalar input: row dimension already resolved by the caller. + theValue = obj; } catch (Exception ex) { log.debug("getDataValue({}): failure: ", index, ex); @@ -620,7 +616,7 @@ private static class CompoundDataProvider extends HDFDataProvider { selectedMemberOrders = compoundFormat.getSelectedMemberOrders(); List localSelectedTypes = - DataFactoryUtils.filterNonSelectedMembers(compoundFormat, dtype); + DataFactoryUtils.filterNonSelectedMembers(compoundFormat, dtype, false); log.trace("setting up {} base HDFDataProviders", localSelectedTypes.size()); @@ -794,6 +790,13 @@ else if (base instanceof ComplexDataProvider) { "CompoundDataProvider.getDataValue: theValue={}, rowIdx={}, adjustedColIndex={}", theValue, rowIdx, adjustedColIndex); } + else if (base instanceof VlenDataProvider) { + // A vlen member is a single column showing the whole sequence. Delegate + // to the provider's full-sequence rendering (an array of per-element + // values that the VlenDataDisplayConverter joins into a brace-string), + // rather than fetching one element by column offset. + theValue = base.getDataValue(colValue, fieldIdx, rowIdx); + } else { log.trace( "CompoundDataProvider.getDataValue: Non-container: Calling base.getDataValue(colValue, {})", @@ -1357,6 +1360,8 @@ private static class VlenDataProvider extends HDFDataProvider { private static final Logger log = LoggerFactory.getLogger(VlenDataProvider.class); private final HDFDataProvider baseTypeDataProvider; + private final Datatype baseType; + private final int numInnerLeaves; private final StringBuilder buffer; @@ -1367,10 +1372,10 @@ private static class VlenDataProvider extends HDFDataProvider { { super(dtype, dataBuf, dataTransposed); - Datatype baseType = dtype.getDatatypeBase(); - baseTypeClass = baseType.getDatatypeClass(); - + baseType = dtype.getDatatypeBase(); + baseTypeClass = baseType.getDatatypeClass(); baseTypeDataProvider = getDataProvider(baseType, dataBuf, dataTransposed); + numInnerLeaves = DataFactoryUtils.countLeafNames(baseType); buffer = new StringBuilder(); } @@ -1476,20 +1481,16 @@ else if (baseTypeDataProvider instanceof ComplexDataProvider) { private Object[] retrieveArrayOfCompoundElements(Object objBuf, int columnIndex, int rowIndex) { - long vlSize = Array.getLength(objBuf); - log.trace("retrieveArrayOfCompoundElements(): vlSize={}", vlSize); - long adjustedRowIdx = - (rowIndex * vlSize * colCount) + - (columnIndex / ((CompoundDataProvider)baseTypeDataProvider).baseProviderIndexMap.size()); - long adjustedColIdx = - columnIndex % ((CompoundDataProvider)baseTypeDataProvider).baseProviderIndexMap.size(); - /* - * Since we flatten array of compound types, we only need to return a single - * value. + * A vlen-of-compound member is one column showing the whole sequence. The read + * path hands each row a list of compound elements already parsed into nested + * Lists (e.g. [[10, [11, 12]], [20, [21, 22]]]). Return the row's elements as an + * array; VlenDataDisplayConverter wraps them in [...] and the inner + * CompoundDataDisplayConverter renders each element as {...} (recursing for + * nested compounds). */ - return new Object[] { - baseTypeDataProvider.getDataValue(objBuf, (int)adjustedColIdx, (int)adjustedRowIdx)}; + ArrayList vlElements = ((ArrayList[])objBuf)[rowIndex]; + return vlElements.toArray(); } private Object[] retrieveArrayOfArrayElements(Object objBuf, int columnIndex, int startRowIndex) @@ -1560,6 +1561,39 @@ private Object[] retrieveArrayOfAtomicElements(Object objBuf, int rowStartIdx) return tempArray; } + /** + * Return one cell value from the row's vlen, given the cell's offset within + * the vlen's column block. The block is laid out element-major: + * {@code [elem0_leaf0, elem0_leaf1, ..., elem1_leaf0, ...]}. Returns null + * when the offset falls past the row's actual vlen length. + */ + Object getElementValue(Object objBuf, int rowIdx, int elementIndex) + { + try { + ArrayList vlElements = ((ArrayList[])objBuf)[rowIdx]; + + if (baseTypeDataProvider instanceof CompoundDataProvider && numInnerLeaves > 0) { + int vlenElemIdx = elementIndex / numInnerLeaves; + int leafIdx = elementIndex % numInnerLeaves; + if (vlenElemIdx < 0 || vlenElemIdx >= vlElements.size()) + return null; + return baseTypeDataProvider.getDataValue(vlElements.get(vlenElemIdx), leafIdx, 0); + } + + if (elementIndex < 0 || elementIndex >= vlElements.size()) + return null; + + Object elem = vlElements.get(elementIndex); + if (elem instanceof byte[]) + return baseTypeDataProvider.getDataValue(elem, 0); + return elem; + } + catch (Exception ex) { + log.debug("getElementValue(rowIdx={}, elementIndex={}): failure: ", rowIdx, elementIndex, ex); + return DataFactoryUtils.errStr; + } + } + @Override public Object getDataValue(Object obj, int index) { diff --git a/hdfview/src/main/java/hdf/view/TableView/DataValidatorFactory.java b/hdfview/src/main/java/hdf/view/TableView/DataValidatorFactory.java index 21ba4ff9..0046b4af 100644 --- a/hdfview/src/main/java/hdf/view/TableView/DataValidatorFactory.java +++ b/hdfview/src/main/java/hdf/view/TableView/DataValidatorFactory.java @@ -244,7 +244,7 @@ private static class CompoundDataValidator extends HDFDataValidator { CompoundDataFormat compoundFormat = (CompoundDataFormat)dataFormatReference; List localSelectedTypes = - DataFactoryUtils.filterNonSelectedMembers(compoundFormat, dtype); + DataFactoryUtils.filterNonSelectedMembers(compoundFormat, dtype, false); log.trace("setting up {} base HDFDataValidators", localSelectedTypes.size()); diff --git a/hdfview/src/main/java/hdf/view/TableView/DefaultBaseTableView.java b/hdfview/src/main/java/hdf/view/TableView/DefaultBaseTableView.java index 71e52eb8..e0501bf6 100644 --- a/hdfview/src/main/java/hdf/view/TableView/DefaultBaseTableView.java +++ b/hdfview/src/main/java/hdf/view/TableView/DefaultBaseTableView.java @@ -105,6 +105,7 @@ import org.eclipse.nebula.widgets.nattable.style.DisplayMode; import org.eclipse.nebula.widgets.nattable.style.HorizontalAlignmentEnum; import org.eclipse.nebula.widgets.nattable.style.Style; +import org.eclipse.nebula.widgets.nattable.ui.action.IKeyAction; import org.eclipse.nebula.widgets.nattable.ui.action.IMouseAction; import org.eclipse.nebula.widgets.nattable.ui.binding.UiBindingRegistry; import org.eclipse.nebula.widgets.nattable.ui.matcher.CellEditorMouseEventMatcher; @@ -119,6 +120,7 @@ import org.eclipse.swt.custom.ScrolledComposite; import org.eclipse.swt.events.DisposeEvent; import org.eclipse.swt.events.DisposeListener; +import org.eclipse.swt.events.KeyEvent; import org.eclipse.swt.events.MouseEvent; import org.eclipse.swt.events.SelectionAdapter; import org.eclipse.swt.events.SelectionEvent; @@ -222,6 +224,14 @@ protected enum ViewType { /** status if file is read only. */ protected boolean isReadOnly = false; + /** + * True when the dataset's datatype contains a construct whose write path + * is not yet symmetric with the display path (see + * {@link DataFactoryUtils#isUnsafeForWrite}). Editing is force-disabled + * and a notice is shown when the user attempts to edit. + */ + protected boolean unsafeForWrite = false; + /** status if the enums are to display converted. */ protected boolean isEnumConverted = false; @@ -412,6 +422,13 @@ public void widgetDisposed(DisposeEvent e) } } + if (!isReadOnly && DataFactoryUtils.isUnsafeForWrite(dataObject.getDatatype())) { + unsafeForWrite = true; + isReadOnly = true; + log.info("dataObject({}) editing disabled: unsafe datatype for table-view write path", + dataObject); + } + log.trace("dataObject({}) isReadOnly={}", dataObject, isReadOnly); long[] dims = dataObject.getDims(); @@ -1278,6 +1295,33 @@ public void updateValueInFile() log.debug("updateValueInFile(): EXIT - value changed flag cleared"); } + /** + * Show the editing-disabled notice. Called when the user attempts to edit + * a cell in a dataset whose datatype has no symmetric write path (see + * {@link DataFactoryUtils#isUnsafeForWrite}). Throttled to one dialog per + * second so a held-down key or rapid clicks don't stack popups. + */ + private long lastUnsafeWriteNoticeMs = 0; + protected final void showUnsafeWriteNotice() + { + if (!unsafeForWrite || shell == null || shell.isDisposed()) + return; + long now = System.currentTimeMillis(); + if (now - lastUnsafeWriteNoticeMs < 1000) + return; + lastUnsafeWriteNoticeMs = now; + + Tools.showInformation(shell, "Editing disabled", + "HDFView does not support editing datasets that contain any of the following:\n" + + " - variable-length sequences (non-string)\n" + + " - arrays of compound, array, or variable-length sequence\n" + + " - arrays nested inside compound\n" + + " - compounds nested inside compound\n\n" + + + "Use a separate tool (h5py, the HDF5 C API, h5edit, etc.) to modify these " + + "datatypes."); + } + @Override public HObject getDataObject() { @@ -2546,6 +2590,35 @@ public void configureUiBindings(UiBindingRegistry uiBindingRegistry) new MouseEditAction()); } }); + + // When the datatype's write path is not yet implemented, intercept + // edit attempts and show a dialog explaining what's blocked. + if (unsafeForWrite) { + this.addConfiguration(new AbstractUiBindingConfiguration() { + @Override + public void configureUiBindings(UiBindingRegistry uiBindingRegistry) + { + uiBindingRegistry.registerFirstDoubleClickBinding( + new MouseEventMatcher(SWT.NONE, GridRegion.BODY, + MouseEventMatcher.LEFT_BUTTON), + new IMouseAction() { + @Override + public void run(NatTable table, MouseEvent event) + { + showUnsafeWriteNotice(); + } + }); + uiBindingRegistry.registerFirstKeyBinding( + new LetterOrDigitKeyEventMatcher(), new IKeyAction() { + @Override + public void run(NatTable table, KeyEvent event) + { + showUnsafeWriteNotice(); + } + }); + } + }); + } } } } diff --git a/hdfview/src/main/java/hdf/view/TableView/DefaultCompoundDSTableView.java b/hdfview/src/main/java/hdf/view/TableView/DefaultCompoundDSTableView.java index cdfe7bf3..b32433ff 100644 --- a/hdfview/src/main/java/hdf/view/TableView/DefaultCompoundDSTableView.java +++ b/hdfview/src/main/java/hdf/view/TableView/DefaultCompoundDSTableView.java @@ -679,6 +679,15 @@ private class CompoundDSCellSelectionListener implements ILayerListener { public void handleLayerEvent(ILayerEvent e) { if (e instanceof CellSelectionEvent) { + // For datatypes where the display column count has been + // expanded (array-of-compound, vlen-of-compound, etc.) the + // listener below hits a null ptr because baseIndexMap was built from the + // unexpanded member count. Editing is already disabled for + // these via unsafeForWrite, and the listener's side effects + // (ref preview, etc.) don't apply, so just return. + if (unsafeForWrite) + return; + CellSelectionEvent event = (CellSelectionEvent)e; boolean valIsRegRef = false; boolean valIsObjRef = false; @@ -840,13 +849,13 @@ private class CompoundDSColumnHeaderDataProvider implements IDataProvider { Datatype cmpdType = dataObject.getDatatype(); - // Resolve VLEN(compound) to the compound base type for display purposes - if (cmpdType.isVLEN() && !cmpdType.isVarStr() && cmpdType.getDatatypeBase() != null && - cmpdType.getDatatypeBase().isCompound()) { - cmpdType = cmpdType.getDatatypeBase(); - } - - List selectedTypes = DataFactoryUtils.filterNonSelectedMembers(dataFormat, cmpdType); + // A top-level vlen-of-compound is a single column (the whole sequence). It is not + // a compound, so it has no members to filter - use it directly as the sole type. + List selectedTypes; + if (cmpdType.isVLEN() && !cmpdType.isVarStr()) + selectedTypes = new ArrayList<>(java.util.Collections.singletonList(cmpdType)); + else + selectedTypes = DataFactoryUtils.filterNonSelectedMembers(dataFormat, cmpdType); final List datasetMemberNames = Arrays.asList(dataFormat.getSelectedMemberNames()); columnNames = new ArrayList<>(dataFormat.getSelectedMemberCount()); @@ -911,7 +920,8 @@ else if (nestedCompoundType.isArray()) { * architectural issue. */ if (memberTypes.isEmpty()) { - memberTypes = DataFactoryUtils.filterNonSelectedMembers(dataFormat, nestedCompoundType); + memberTypes = + DataFactoryUtils.filterNonSelectedMembers(dataFormat, nestedCompoundType, false); } /* @@ -945,33 +955,37 @@ else if (nestedCompoundType.isArray()) { memberTypes); } else if (curDtype.isVLEN() && !curDtype.isVarStr()) { - /* - * For VLEN of COMPOUND, peel off the VLEN wrapper and recurse with the - * compound base type. Each cell displays the variable-length values as a - * brace-enclosed list, so no column multiplication is needed. - */ - Datatype baseType = curDtype.getDatatypeBase(); - if (baseType != null && baseType.isCompound()) { - if (memberTypes.isEmpty()) { - memberTypes = DataFactoryUtils.filterNonSelectedMembers(dataFormat, baseType); - } - recursiveColumnHeaderSetup(outColNames, dataFormat, baseType, memberNames, memberTypes); - } + // A top-level vlen (including vlen-of-compound) is a single column showing + // the whole sequence. Never recurse into a compound base - that would create + // one column per inner member. + for (int j = 0; j < memberNames.size(); j++) + outColNames.add(memberNames.get(j).replaceAll(CompoundDS.SEPARATOR, "->")); } else if (curDtype.isCompound()) { + /* + * memberNames is the flat list of leaf names; memberTypes is the list of + * top-level member types. Advance through memberTypes one slot at a time, + * consuming countLeafNames(topType) flat names per slot. + */ ListIterator localIt = memberNames.listIterator(); + int topIdx = 0; + int remainingLeavesInTop = DataFactoryUtils.countLeafNames(memberTypes.get(0)); while (localIt.hasNext()) { - int curIdx = localIt.nextIndex(); + if (remainingLeavesInTop <= 0 && topIdx + 1 < memberTypes.size()) { + topIdx++; + remainingLeavesInTop = DataFactoryUtils.countLeafNames(memberTypes.get(topIdx)); + } String curName = localIt.next(); - Datatype curType = memberTypes.get(curIdx % memberTypes.size()); + Datatype curType = memberTypes.get(topIdx); Datatype nestedArrayOfCompoundType = null; boolean nestedArrayOfCompound = false; /* - * Recursively detect any nested array/vlen of compound types and deal with them - * by creating multiple copies of the member names. + * Recursively detect any nested ARRAY of compound types and deal with them + * by creating multiple copies of the member names. A vlen is NOT expanded - + * it is a single column (handled below), so it is excluded here. */ - if (curType.isArray() || curType.isVLEN()) { + if (curType.isArray()) { Datatype base = curType.getDatatypeBase(); while (base != null) { if (base.isCompound()) { @@ -989,29 +1003,85 @@ else if (curDtype.isCompound()) { * members n times, where n is the number of array or vlen elements. */ if (nestedArrayOfCompound) { - List selTypes = - DataFactoryUtils.filterNonSelectedMembers(dataFormat, nestedArrayOfCompoundType); - List selMemberNames = new ArrayList<>(selTypes.size()); - - int arrCmpdLen = calcArrayOfCompoundLen(selTypes); - for (int i = 0; i < arrCmpdLen; i++) { - selMemberNames.add(localIt.next()); + List selTypes = DataFactoryUtils.filterNonSelectedMembers( + dataFormat, nestedArrayOfCompoundType, false); + + List selMemberNames; + int namesConsumed; + if (curType.isVLEN()) { + // Vlen wraps the flat name list at the header level only; + // synthesize the inner-compound leaf names from the declared + // member-name list, using the header as prefix. + String baseName = curName.replaceAll(CompoundDS.SEPARATOR, "->"); + selMemberNames = buildInnerCompoundLeafNames(nestedArrayOfCompoundType, baseName); + namesConsumed = 1; + } + else { + // Array-of-compound: flat list has header + each inner leaf name. + selMemberNames = new ArrayList<>(selTypes.size()); + int arrCmpdLen = calcArrayOfCompoundLen(selTypes); + selMemberNames.add(curName); + for (int i = 1; i < arrCmpdLen; i++) + selMemberNames.add(localIt.next()); + namesConsumed = arrCmpdLen; } recursiveColumnHeaderSetup(outColNames, dataFormat, curType, selMemberNames, selTypes); + remainingLeavesInTop -= namesConsumed; + } + else if (curType.isVLEN() && !curType.isVarStr()) { + // A vlen member is a single column showing the whole sequence as a + // brace-string (VlenDataProvider recurses for nested compounds). + String baseName = curName.replaceAll(CompoundDS.SEPARATOR, "->"); + outColNames.add(baseName); + remainingLeavesInTop--; } else { // Copy the dataset member name reference, so changes to the column name // don't affect the dataset's internal member names. curName = new String(curName.replaceAll(CompoundDS.SEPARATOR, "->")); - outColNames.add(curName); + remainingLeavesInTop--; } } } } + /** + * Produce "prefix->leaf" names for each leaf of {@code innerCompound}. + * Used by the vlen-of-compound header path, whose flat name list contains + * only the wrapper header rather than per-leaf entries. + */ + private List buildInnerCompoundLeafNames(Datatype innerCompound, String prefix) + { + List out = new ArrayList<>(); + appendInnerCompoundLeafNames(innerCompound, prefix, out); + return out; + } + + private void appendInnerCompoundLeafNames(Datatype t, String prefix, List out) + { + if (t == null) + return; + if (t.isCompound()) { + List names = t.getCompoundMemberNames(); + List children = t.getCompoundMemberTypes(); + if (names == null || children == null) + return; + for (int i = 0; i < names.size(); i++) { + String childName = prefix + "->" + names.get(i); + Datatype childT = children.get(i); + if (childT != null && childT.isCompound()) + appendInnerCompoundLeafNames(childT, childName, out); + else + out.add(childName); + } + return; + } + out.add(prefix); + } + private int calcArrayOfCompoundLen(List datatypes) { int count = 0; @@ -1157,10 +1227,12 @@ else if (groupTitleStartPosition > 0) { } else if (allColumnNames[i].matches(".*\\[[0-9]*\\]")) { /* - * Top-level ARRAY of COMPOUND types. + * Top-level array/vlen of an atomic base. Each element is a single + * column, so group them all under the member name rather than a + * generic per-element "ARRAY[i]" group. */ - columnHeaderBuilder.append("ARRAY"); - processArrayOfCompound(columnHeaderBuilder, allColumnNames[i]); + String baseName = allColumnNames[i].replaceAll("\\[[0-9]*\\]$", ""); + columnHeaderBuilder.append(baseName); columnGroupHeaderLayer.addColumnsIndexesToGroup(columnHeaderBuilder.toString(), colindex); diff --git a/object/src/main/java/hdf/object/h5/H5CompoundDS.java b/object/src/main/java/hdf/object/h5/H5CompoundDS.java index d8be2f4b..e9c34929 100644 --- a/object/src/main/java/hdf/object/h5/H5CompoundDS.java +++ b/object/src/main/java/hdf/object/h5/H5CompoundDS.java @@ -944,14 +944,41 @@ private Object compoundTypeIO(H5File.IO_TYPE ioType, long did, long[] spaceIDs, } else if (cmpdType.isVLEN() && !cmpdType.isVarStr()) { /* - * For VLEN of COMPOUND, peel off the VLEN wrapper and recurse with the compound - * base type. The actual VLEN reading is handled in readSingleCompoundMember(), - * which detects the VLEN wrapper via dsDatatype.isVLEN() and uses H5DreadVL. + * Top-level VLEN-of-compound is displayed as a SINGLE column showing the whole + * sequence (e.g. [{1, 2}, {3, 4}]), consistent with how a vlen member of a + * compound is shown. Read the whole vlen in one H5DreadVL call; the JNI parses + * each compound element into nested Lists (e.g. [[1, 2], [3, 4]]), exactly the + * shape VlenDataProvider/VlenDataDisplayConverter render. The dataset enumerates + * a single member (see H5Datatype.extractCompoundInfo), so we return one column. */ - Datatype baseType = cmpdType.getDatatypeBase(); - if (baseType != null && baseType.isCompound()) { - theData = compoundTypeIO(ioType, did, spaceIDs, nSelPoints, (H5Datatype)baseType, writeBuf, - globalMemberIndex); + if (ioType == H5File.IO_TYPE.READ) { + long wholeTid = -1; + try { + wholeTid = H5.H5Dget_type(did); + @SuppressWarnings("rawtypes") + ArrayList[] vlBuf = new ArrayList[nSelPoints]; + H5.H5DreadVL(did, wholeTid, spaceIDs[0], spaceIDs[1], HDF5Constants.H5P_DEFAULT, vlBuf); + globalMemberIndex[0]++; + theData = vlBuf; + } + catch (HDF5DataFiltersException exfltr) { + log.debug("compoundTypeIO(): top-level VLEN read failure: ", exfltr); + throw new HDF5Exception("Filter not available exception: " + exfltr.getMessage()); + } + catch (Exception ex) { + log.debug("compoundTypeIO(): top-level VLEN read failure: ", ex); + throw new HDF5Exception("failed to read VLEN-of-compound dataset: " + ex.getMessage()); + } + finally { + if (wholeTid >= 0) { + try { + H5.H5Tclose(wholeTid); + } + catch (Exception ex) { + log.debug("compoundTypeIO(): H5Tclose(wholeTid {}) failure: ", wholeTid, ex); + } + } + } } } else if (cmpdType.isCompound()) { @@ -1214,10 +1241,10 @@ private Object readSingleCompoundMember(long dsetID, long[] spaceIDs, int nSelPo (spaceIDs[0] == HDF5Constants.H5P_DEFAULT) ? "H5P_DEFAULT" : spaceIDs[0], (spaceIDs[1] == HDF5Constants.H5P_DEFAULT) ? "H5P_DEFAULT" : spaceIDs[1]); + // Slots are left null: H5DreadVL installs a freshly + // allocated ArrayList into each slot. @SuppressWarnings("rawtypes") ArrayList[] vlBuf = new ArrayList[nSelPoints]; - for (int j = 0; j < nSelPoints; j++) - vlBuf[j] = new ArrayList<>(); H5.H5DreadVL(dsetID, compTid, spaceIDs[0], spaceIDs[1], HDF5Constants.H5P_DEFAULT, vlBuf); @@ -1262,6 +1289,17 @@ else if (memberType.isVLEN() || H5.H5DreadVL(dsetID, compTid, spaceIDs[0], spaceIDs[1], HDF5Constants.H5P_DEFAULT, (Object[])memberData); + + // H5DreadVL was called with a single-field compound transfer type, so + // each row comes back wrapped in a one-element record. Unwrap to expose + // the payload directly. + if (memberType.isVLEN()) { + Object[] rows = (Object[])memberData; + for (int r = 0; r < rows.length; r++) { + if (rows[r] instanceof java.util.ArrayList rec && rec.size() == 1) + rows[r] = rec.get(0); + } + } } else { log.trace( diff --git a/object/src/main/java/hdf/object/h5/H5Datatype.java b/object/src/main/java/hdf/object/h5/H5Datatype.java index 306ffba1..ddd37689 100644 --- a/object/src/main/java/hdf/object/h5/H5Datatype.java +++ b/object/src/main/java/hdf/object/h5/H5Datatype.java @@ -2166,11 +2166,10 @@ else if ((typeClass == CLASS_STRING) || (typeClass == CLASS_REFERENCE)) { else if (dtype.isVLEN()) { log.trace("allocateArray(): isVLEN"); + // Slots are left null: the JNI read routines (H5DreadVL/H5AreadVL) + // install a freshly allocated ArrayList into each slot. Callers must + // not pre-allocate the per-element lists. data = new ArrayList[numPoints]; - for (int j = 0; j < numPoints; j++) - ((ArrayList[])data)[j] = new ArrayList(); - // if (baseType != null) - // ((ArrayList<>)data).add(H5Datatype.allocateArray(baseType, numPoints)); } else if (typeClass == CLASS_ARRAY) { log.trace("allocateArray(): class CLASS_ARRAY"); @@ -2975,9 +2974,15 @@ public static void extractCompoundInfo(final H5Datatype dtype, String name, List H5Datatype.extractCompoundInfo((H5Datatype)dtype.getDatatypeBase(), name, names, flatListTypes); } else if (dtype.isVLEN() && !dtype.isVarStr()) { - log.trace( - "extractCompoundInfo(): variable-length type - extracting compound info from base datatype"); - H5Datatype.extractCompoundInfo((H5Datatype)dtype.getDatatypeBase(), name, names, flatListTypes); + /* + * A vlen (including vlen-of-compound) is a single leaf: it is displayed as one + * column showing the whole sequence as a string. Do NOT recurse into the base + * compound - that would enumerate the inner members as separate columns. + */ + log.trace("extractCompoundInfo(): variable-length type - adding as a single leaf"); + if (names != null) + names.add(name); + flatListTypes.add(dtype); } else if (dtype.isCompound()) { List compoundMemberNames = dtype.getCompoundMemberNames(); diff --git a/object/src/main/java/hdf/object/h5/H5ScalarAttr.java b/object/src/main/java/hdf/object/h5/H5ScalarAttr.java index 791d2c86..96dcad4f 100644 --- a/object/src/main/java/hdf/object/h5/H5ScalarAttr.java +++ b/object/src/main/java/hdf/object/h5/H5ScalarAttr.java @@ -1210,9 +1210,9 @@ else if (dsDatatype.isCompound()) { else if (dsDatatype.isVLEN()) { log.trace("attributeCommonIO():read ioType:VLEN-REF H5Aread isArray()={}", dsDatatype.isArray()); + // Slots are left null: H5AreadVL installs a freshly + // allocated ArrayList into each slot. theData = new ArrayList[(int)lsize]; - for (int j = 0; j < lsize; j++) - ((ArrayList[])theData)[j] = new ArrayList(); try { H5.H5AreadVL(attrID, tid, (Object[])theData); diff --git a/object/src/main/java/hdf/object/h5/H5ScalarDS.java b/object/src/main/java/hdf/object/h5/H5ScalarDS.java index f36a0b50..4af59457 100644 --- a/object/src/main/java/hdf/object/h5/H5ScalarDS.java +++ b/object/src/main/java/hdf/object/h5/H5ScalarDS.java @@ -940,9 +940,9 @@ private Object scalarDatasetCommonIO(H5File.IO_TYPE ioType, Object writeBuf) thr } } else if (dsDatatype.isVLEN()) { + // Slots are left null: H5DreadVL installs a freshly + // allocated ArrayList into each slot. theData = new ArrayList[(int)totalSelectedSpacePoints]; - for (int j = 0; j < (int)totalSelectedSpacePoints; j++) - ((ArrayList[])theData)[j] = new ArrayList(); } else if ((originalBuf == null) || dsDatatype.isEnum() || dsDatatype.isText() || dsDatatype.isRefObj() || @@ -974,7 +974,9 @@ else if ((originalBuf == null) || dsDatatype.isEnum() || dsDatatype.isText() || tid = dsDatatype.createNative(); log.trace("scalarDatasetCommonIO(): native type created tid={}", tid); - if (dsDatatype.isVarStr()) { + if (dsDatatype.isVarStr() || + (dsDatatype.isArray() && dsDatatype.getDatatypeBase() != null && + dsDatatype.getDatatypeBase().isVarStr())) { log.trace( "scalarDatasetCommonIO(): H5Dread_VLStrings did={} tid={} spaceIDs[0]={} spaceIDs[1]={}", did, tid, @@ -1125,7 +1127,9 @@ else if (dsDatatype.isFloat() && dsDatatype.getDatatypeSize() == 16) { try { tid = dsDatatype.createNative(); - if (dsDatatype.isVarStr()) { + if (dsDatatype.isVarStr() || + (dsDatatype.isArray() && dsDatatype.getDatatypeBase() != null && + dsDatatype.getDatatypeBase().isVarStr())) { log.trace( "scalarDatasetCommonIO(): H5Dwrite_VLStrings did={} tid={} spaceIDs[0]={} spaceIDs[1]={}", did, tid,