From b340f0ba52fa8e05376937b67fc00fe76b293848 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 20 Jul 2026 20:01:08 -0400 Subject: [PATCH 01/15] Expose H5Dchunk_iter, H5Dget_num_chunks, H5Dget_chunk_info in JNI Java bindings Adds native declarations and JNI glue for H5Dchunk_iter (with a new H5D_chunk_iter_cb/H5D_chunk_iter_t callback pair) plus the by-index chunk inspection functions H5Dget_num_chunks and H5Dget_chunk_info, enabling Java callers to enumerate chunks without a per-tool JNI helper. Includes JUnit coverage and a quick benchmark comparing H5Dchunk_iter against a per-index H5Dget_chunk_info loop, mirroring the scaling analysis from https://github.com/JuliaIO/HDF5.jl/pull/1031#issuecomment-1407749686. --- java/src-jni/hdf/hdf5lib/CMakeLists.txt | 2 + java/src-jni/hdf/hdf5lib/H5.java | 82 ++++++++ .../hdf5lib/callbacks/H5D_chunk_iter_cb.java | 40 ++++ .../hdf5lib/callbacks/H5D_chunk_iter_t.java | 24 +++ java/src-jni/jni/h5dImp.c | 184 ++++++++++++++++++ java/src-jni/jni/h5dImp.h | 23 +++ java/src-jni/test/CMakeLists.txt | 1 + java/src-jni/test/TestH5D.java | 49 +++++ java/src-jni/test/TestH5DChunkIterPerf.java | 142 ++++++++++++++ 9 files changed, 547 insertions(+) create mode 100644 java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java create mode 100644 java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java create mode 100644 java/src-jni/test/TestH5DChunkIterPerf.java diff --git a/java/src-jni/hdf/hdf5lib/CMakeLists.txt b/java/src-jni/hdf/hdf5lib/CMakeLists.txt index 9db942db4ad5..c5d5f4c9c358 100644 --- a/java/src-jni/hdf/hdf5lib/CMakeLists.txt +++ b/java/src-jni/hdf/hdf5lib/CMakeLists.txt @@ -20,6 +20,8 @@ set (HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES callbacks/H5A_iterate_t.java callbacks/H5D_append_cb.java callbacks/H5D_append_t.java + callbacks/H5D_chunk_iter_cb.java + callbacks/H5D_chunk_iter_t.java callbacks/H5D_iterate_cb.java callbacks/H5D_iterate_t.java callbacks/H5E_walk_cb.java diff --git a/java/src-jni/hdf/hdf5lib/H5.java b/java/src-jni/hdf/hdf5lib/H5.java index 029318e3e00c..ca85d134cd42 100644 --- a/java/src-jni/hdf/hdf5lib/H5.java +++ b/java/src-jni/hdf/hdf5lib/H5.java @@ -19,6 +19,8 @@ import hdf.hdf5lib.callbacks.H5A_iterate_cb; import hdf.hdf5lib.callbacks.H5A_iterate_t; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; import hdf.hdf5lib.callbacks.H5D_iterate_cb; import hdf.hdf5lib.callbacks.H5D_iterate_t; import hdf.hdf5lib.callbacks.H5E_walk_cb; @@ -2777,6 +2779,33 @@ public static long H5Dget_type(long dataset_id) throws HDF5LibraryException private synchronized static native long _H5Dget_type(long dataset_id) throws HDF5LibraryException; + /** + * @ingroup JH5D + * + * H5Dchunk_iter iterates over all chunks in the dataset, calling the user supplied callback with the + * details of the chunk and the supplied context op_data. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * @param op + * IN: Callback function to operate on each chunk. + * @param op_data + * IN/OUT: Pointer to any user-defined data for use by operator function. + * + * @return returns the return value of the first operator that returns a positive value, or zero if all + * chunks were processed with no operator returning non-zero. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * op is null. + **/ + public synchronized static native int H5Dchunk_iter(long dataset_id, long dxpl_id, H5D_chunk_iter_cb op, + H5D_chunk_iter_t op_data) + throws HDF5LibraryException, NullPointerException; + /** * @ingroup JH5D * @@ -4244,6 +4273,59 @@ public synchronized static native int H5Dwrite_VLStrings(long dataset_id, long m **/ public synchronized static native void H5Drefresh(long dset_id) throws HDF5LibraryException; + /** + * @ingroup JH5D + * + * H5Dget_num_chunks retrieves the number of chunks that have a nonempty intersection with the + * selection specified by fspace_id. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param fspace_id + * IN: File dataspace selection identifier. + * + * @return the number of chunks + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native long H5Dget_num_chunks(long dataset_id, long fspace_id) + throws HDF5LibraryException; + + /** + * @ingroup JH5D + * + * H5Dget_chunk_info retrieves the offset, filter mask, address, and size for the chunk specified + * by its index chk_idx within the selection specified by fspace_id. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param fspace_id + * IN: File dataspace selection identifier. + * @param chk_idx + * IN: Index of the chunk. + * @param offset + * OUT: Array of size equal to the dataset's rank; filled with the logical position of + * the chunk's first element in each dimension. + * @param filter_mask + * OUT: Array of size one; filled with the bitmask indicating the filters used when the + * chunk was written. + * @param addr + * OUT: Array of size one; filled with the chunk address in the file. + * @param size + * OUT: Array of size one; filled with the chunk size in bytes, 0 if the chunk does not + * exist. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * an output array is null. + **/ + public synchronized static native void H5Dget_chunk_info(long dataset_id, long fspace_id, long chk_idx, + long[] offset, int[] filter_mask, long[] addr, + long[] size) + throws HDF5LibraryException, NullPointerException; + // /////// unimplemented //////// // herr_t H5Ddebug(hid_t dset_id); // herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); diff --git a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java new file mode 100644 index 000000000000..6a4e7390ae18 --- /dev/null +++ b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java @@ -0,0 +1,40 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package hdf.hdf5lib.callbacks; + +/** + * Information class for link callback for H5Dchunk_iter. + * + */ +public interface H5D_chunk_iter_cb extends H5Callbacks { + /** + * @ingroup JCALLBK + * + * application callback for each chunk of a chunked dataset + * + * @param offset the logical position of the chunk's first element in units of dataset elements + * @param filter_mask bitmask indicating the filters used when the chunk was written + * @param addr the chunk address in the file, taking the user block (if any) into account + * @param size the chunk size in bytes, 0 if the chunk does not exist + * @param op_data the operator data passed in to H5Dchunk_iter + * + * @return operation status + * A. Zero causes the iterator to continue, returning zero when all + * chunks have been processed. + * B. Positive causes the iterator to immediately return that positive + * value, indicating short-circuit success. + * C. Negative causes the iterator to immediately return that value, + * indicating failure. + */ + int callback(long[] offset, int filter_mask, long addr, long size, H5D_chunk_iter_t op_data); +} diff --git a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java new file mode 100644 index 000000000000..488640272779 --- /dev/null +++ b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java @@ -0,0 +1,24 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package hdf.hdf5lib.callbacks; + +/** + * Data class for link callback for H5Dchunk_iter. + * + */ +public interface H5D_chunk_iter_t { + /** + * public ArrayList iterdata = new ArrayList(); + * Any derived interfaces must define the single public variable as above. + */ +} diff --git a/java/src-jni/jni/h5dImp.c b/java/src-jni/jni/h5dImp.c index 421668a03466..b7842ab88acb 100644 --- a/java/src-jni/jni/h5dImp.c +++ b/java/src-jni/jni/h5dImp.c @@ -33,6 +33,12 @@ typedef struct _cb_wrapper { jobject op_data; } cb_wrapper; +typedef struct _chunk_iter_cb_wrapper { + jobject visit_callback; + jobject op_data; + unsigned rank; +} chunk_iter_cb_wrapper; + /********************/ /* Local Prototypes */ /********************/ @@ -2079,6 +2085,94 @@ Java_hdf_hdf5lib_H5_H5Dset_1extent(JNIEnv *env, jclass clss, jlong loc_id, jlong UNPIN_LONG_ARRAY(ENVONLY, buf, dimsBuf, JNI_ABORT); } /* end Java_hdf_hdf5lib_H5_H5Dset_1extent */ +static herr_t +H5D_chunk_iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *cb_data) +{ + chunk_iter_cb_wrapper *wrapper = (chunk_iter_cb_wrapper *)cb_data; + jlongArray offsetArray; + jmethodID mid; + jobject visit_callback = wrapper->visit_callback; + jclass cls; + JNIEnv *cbenv = NULL; + jint status = FAIL; + void *op_data = (void *)wrapper->op_data; + + if (JVMPTR->AttachCurrentThread(JVMPAR, (void **)&cbenv, NULL) < 0) { + CHECK_JNI_EXCEPTION(CBENVONLY, JNI_TRUE); + H5_JNI_FATAL_ERROR(CBENVONLY, "H5D_chunk_iter_cb: failed to attach current thread to JVM"); + } + + if (NULL == (cls = CBENVPTR->GetObjectClass(CBENVONLY, visit_callback))) + CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); + + if (NULL == (mid = CBENVPTR->GetMethodID(CBENVONLY, cls, "callback", + "([JIJJLhdf/hdf5lib/callbacks/H5D_chunk_iter_t;)I"))) + CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); + + if (NULL == offset) + H5_NULL_ARGUMENT_ERROR(CBENVONLY, "H5D_chunk_iter_cb: offset is NULL"); + + if (NULL == (offsetArray = CBENVPTR->NewLongArray(CBENVONLY, (jsize)wrapper->rank))) + CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); + + CBENVPTR->SetLongArrayRegion(CBENVONLY, offsetArray, 0, (jsize)wrapper->rank, (const jlong *)offset); + CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); + + status = CBENVPTR->CallIntMethod(CBENVONLY, visit_callback, mid, offsetArray, (jint)filter_mask, + (jlong)addr, (jlong)size, op_data); + CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); + +done: + if (cbenv) + JVMPTR->DetachCurrentThread(JVMPAR); + + return (herr_t)status; +} /* end H5D_chunk_iter_cb */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dchunk_iter + * Signature: (JJLjava/lang/Object;Ljava/lang/Object;)I + */ +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Dchunk_1iter(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id, + jobject callback_op, jobject op_data) +{ + chunk_iter_cb_wrapper wrapper = {callback_op, op_data, 0}; + hid_t space_id = H5I_INVALID_HID; + bool close_space = false; + int ndims; + herr_t status = FAIL; + + UNUSED(clss); + + ENVPTR->GetJavaVM(ENVONLY, &jvm); + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + if (NULL == callback_op) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dchunk_iter: callback_op is NULL"); + if (NULL == op_data) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dchunk_iter: op_data is NULL"); + + if ((space_id = H5Dget_space((hid_t)dataset_id)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + close_space = true; + + if ((ndims = H5Sget_simple_extent_ndims(space_id)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + wrapper.rank = (unsigned)ndims; + + if ((status = H5Dchunk_iter((hid_t)dataset_id, (hid_t)dxpl_id, (H5D_chunk_iter_op_t)H5D_chunk_iter_cb, + (void *)&wrapper)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + if (close_space) + H5Sclose(space_id); + + return (jint)status; +} /* end Java_hdf_hdf5lib_H5_H5Dchunk_1iter */ + static herr_t H5D_iterate_cb(void *elem, hid_t elem_id, unsigned ndim, const hsize_t *point, void *cb_data) { @@ -2213,6 +2307,96 @@ Java_hdf_hdf5lib_H5_H5Drefresh(JNIEnv *env, jclass clss, jlong loc_id) return; } +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_num_chunks + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Dget_1num_1chunks(JNIEnv *env, jclass clss, jlong dataset_id, jlong fspace_id) +{ + hsize_t nchunks = 0; + herr_t status = FAIL; + + UNUSED(clss); + + if ((status = H5Dget_num_chunks((hid_t)dataset_id, (hid_t)fspace_id, &nchunks)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return (jlong)nchunks; +} /* end Java_hdf_hdf5lib_H5_H5Dget_1num_1chunks */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_info + * Signature: (JJJ[J[I[J[J)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *env, jclass clss, jlong dataset_id, jlong fspace_id, + jlong chk_idx, jlongArray offset, jintArray filter_mask, + jlongArray addr, jlongArray size) +{ + jboolean isCopy; + jlong *offsetArray = NULL; + jint *maskArray = NULL; + jlong *addrArray = NULL; + jlong *sizeArray = NULL; + hsize_t *offsetBuf = NULL; + hsize_t size_val = 0; + haddr_t addr_val = HADDR_UNDEF; + unsigned filter_mask_val = 0; + jsize rank; + jsize i; + herr_t status = FAIL; + + UNUSED(clss); + + if (NULL == offset) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info: offset is NULL"); + if (NULL == filter_mask) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info: filter_mask is NULL"); + if (NULL == addr) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info: addr is NULL"); + if (NULL == size) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info: size is NULL"); + + if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + + if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dget_chunk_info: failed to allocate offset buffer"); + + if ((status = H5Dget_chunk_info((hid_t)dataset_id, (hid_t)fspace_id, (hsize_t)chk_idx, offsetBuf, + &filter_mask_val, &addr_val, &size_val)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, "H5Dget_chunk_info: offset not pinned"); + for (i = 0; i < rank; i++) + offsetArray[i] = (jlong)offsetBuf[i]; + + PIN_INT_ARRAY(ENVONLY, filter_mask, maskArray, &isCopy, "H5Dget_chunk_info: filter_mask not pinned"); + maskArray[0] = (jint)filter_mask_val; + + PIN_LONG_ARRAY(ENVONLY, addr, addrArray, &isCopy, "H5Dget_chunk_info: addr not pinned"); + addrArray[0] = (jlong)addr_val; + + PIN_LONG_ARRAY(ENVONLY, size, sizeArray, &isCopy, "H5Dget_chunk_info: size not pinned"); + sizeArray[0] = (jlong)size_val; + +done: + if (offsetBuf) + free(offsetBuf); + if (sizeArray) + UNPIN_LONG_ARRAY(ENVONLY, size, sizeArray, (status < 0) ? JNI_ABORT : 0); + if (addrArray) + UNPIN_LONG_ARRAY(ENVONLY, addr, addrArray, (status < 0) ? JNI_ABORT : 0); + if (maskArray) + UNPIN_INT_ARRAY(ENVONLY, filter_mask, maskArray, (status < 0) ? JNI_ABORT : 0); + if (offsetArray) + UNPIN_LONG_ARRAY(ENVONLY, offset, offsetArray, (status < 0) ? JNI_ABORT : 0); +} /* end Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info */ + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src-jni/jni/h5dImp.h b/java/src-jni/jni/h5dImp.h index 7b1916c48c18..f8e32c02b814 100644 --- a/java/src-jni/jni/h5dImp.h +++ b/java/src-jni/jni/h5dImp.h @@ -296,6 +296,13 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dfill(JNIEnv *, jclass, jbyteArray, */ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dset_1extent(JNIEnv *, jclass, jlong, jlongArray); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dchunk_iter + * Signature: (JJLjava/lang/Object;Ljava/lang/Object;)I + */ +JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dchunk_1iter(JNIEnv *, jclass, jlong, jlong, jobject, jobject); + /* * Class: hdf_hdf5lib_H5 * Method: H5Diterate @@ -318,6 +325,22 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dflush(JNIEnv *, jclass, jlong); */ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Drefresh(JNIEnv *, jclass, jlong); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_num_chunks + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Dget_1num_1chunks(JNIEnv *, jclass, jlong, jlong); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_info + * Signature: (JJJ[J[I[J[J)V + */ +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *, jclass, jlong, jlong, jlong, + jlongArray, jintArray, jlongArray, + jlongArray); + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src-jni/test/CMakeLists.txt b/java/src-jni/test/CMakeLists.txt index d23894831836..722a08c9bb4f 100644 --- a/java/src-jni/test/CMakeLists.txt +++ b/java/src-jni/test/CMakeLists.txt @@ -30,6 +30,7 @@ set (HDF5_JAVA_TEST_SOURCES TestH5D TestH5Dplist TestH5Drw + TestH5DChunkIterPerf TestH5Lparams TestH5Lbasic TestH5Lcreate diff --git a/java/src-jni/test/TestH5D.java b/java/src-jni/test/TestH5D.java index 18e33fa6600c..bb8de406491e 100644 --- a/java/src-jni/test/TestH5D.java +++ b/java/src-jni/test/TestH5D.java @@ -26,6 +26,8 @@ import hdf.hdf5lib.H5; import hdf.hdf5lib.HDF5Constants; import hdf.hdf5lib.HDFNativeData; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; import hdf.hdf5lib.callbacks.H5D_iterate_cb; import hdf.hdf5lib.callbacks.H5D_iterate_t; import hdf.hdf5lib.exceptions.HDF5Exception; @@ -754,6 +756,53 @@ public void testH5Dfill() buf_data[(indx * DIM_Y) + jndx] == 254); } + @Test + public void testH5Dchunk_iter() + { + final int[] chunk_count = {0}; + + class H5D_chunk_iter_data implements H5D_chunk_iter_t { + } + + class H5D_chunk_iter_callback implements H5D_chunk_iter_cb { + public int callback(long[] offset, int filter_mask, long addr, long size, H5D_chunk_iter_t op_data) + { + assertEquals("testH5Dchunk_iter: offset rank", RANK, offset.length); + chunk_count[0]++; + return 0; + } + } + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_iter", HDF5Constants.H5P_DEFAULT); + + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + H5D_chunk_iter_cb iter_cb = new H5D_chunk_iter_callback(); + H5D_chunk_iter_t iter_data = new H5D_chunk_iter_data(); + int op_status = -1; + try { + op_status = H5.H5Dchunk_iter(H5did, HDF5Constants.H5P_DEFAULT, iter_cb, iter_data); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dchunk_iter: " + err); + } + assertTrue("H5Dchunk_iter ", op_status == 0); + assertEquals("testH5Dchunk_iter: chunk count", 2, chunk_count[0]); + } + @Test public void testH5Diterate() { diff --git a/java/src-jni/test/TestH5DChunkIterPerf.java b/java/src-jni/test/TestH5DChunkIterPerf.java new file mode 100644 index 000000000000..a3b26e29fe79 --- /dev/null +++ b/java/src-jni/test/TestH5DChunkIterPerf.java @@ -0,0 +1,142 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package test; + +import static org.junit.Assert.assertEquals; + +import java.io.File; + +import hdf.hdf5lib.H5; +import hdf.hdf5lib.HDF5Constants; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Quick benchmark comparing H5Dchunk_iter against looping over H5Dget_chunk_info by + * chunk index, following the methodology from + * https://github.com/JuliaIO/HDF5.jl/pull/1031#issuecomment-1407749686 which found that + * H5Dchunk_iter scales much better than repeated by-index lookups as the chunk count grows. + */ +public class TestH5DChunkIterPerf { + private static final String H5_FILE = "testDChunkIterPerf.h5"; + private static final int CHUNK = 16; + private static final int[] SIZES = {32, 64, 128, 256, 512, 1024}; + + long H5fid = HDF5Constants.H5I_INVALID_HID; + + private final void _deleteFile(String filename) + { + File file = new File(filename); + if (file.exists()) { + try { + file.delete(); + } + catch (SecurityException e) { + } + } + } + + @Before + public void createH5file() throws Exception + { + _deleteFile(H5_FILE); + H5fid = H5.H5Fcreate(H5_FILE, HDF5Constants.H5F_ACC_TRUNC, HDF5Constants.H5P_DEFAULT, + HDF5Constants.H5P_DEFAULT); + } + + @After + public void deleteH5file() throws Exception + { + if (H5fid >= 0) + H5.H5Fclose(H5fid); + _deleteFile(H5_FILE); + } + + private long createChunkedDataset(int size) throws Exception + { + long dcpl_id = H5.H5Pcreate(HDF5Constants.H5P_DATASET_CREATE); + H5.H5Pset_alloc_time(dcpl_id, HDF5Constants.H5D_ALLOC_TIME_EARLY); + H5.H5Pset_chunk(dcpl_id, 2, new long[] {CHUNK, CHUNK}); + + long sid = H5.H5Screate_simple(2, new long[] {size, size}, null); + long did = H5.H5Dcreate(H5fid, "dset" + size, HDF5Constants.H5T_NATIVE_UINT8, sid, + HDF5Constants.H5P_DEFAULT, dcpl_id, HDF5Constants.H5P_DEFAULT); + H5.H5Pclose(dcpl_id); + H5.H5Sclose(sid); + return did; + } + + private long countByIterate(long did) throws Exception + { + final long[] count = {0}; + class Data implements H5D_chunk_iter_t { + } + H5D_chunk_iter_cb cb = new H5D_chunk_iter_cb() { + public int callback(long[] offset, int filter_mask, long addr, long size, H5D_chunk_iter_t op_data) + { + count[0]++; + return 0; + } + }; + H5.H5Dchunk_iter(did, HDF5Constants.H5P_DEFAULT, cb, new Data()); + return count[0]; + } + + private long countByIndex(long did) throws Exception + { + long sid = H5.H5Dget_space(did); + long nchunks = H5.H5Dget_num_chunks(did, sid); + long[] offset = new long[2]; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; + + for (long i = 0; i < nchunks; i++) + H5.H5Dget_chunk_info(did, sid, i, offset, filter_mask, addr, size); + + H5.H5Sclose(sid); + return nchunks; + } + + @Test + public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception + { + System.out.println(); + System.out.printf("%10s %14s %14s %10s%n", "size", "iter_ms", "indx_ms", "ratio"); + + for (int size : SIZES) { + long did = createChunkedDataset(size); + + long t0 = System.nanoTime(); + long count_iter = countByIterate(did); + long t1 = System.nanoTime(); + long count_index = countByIndex(did); + long t2 = System.nanoTime(); + + H5.H5Dclose(did); + + double iter_ms = (t1 - t0) / 1.0e6; + double indx_ms = (t2 - t1) / 1.0e6; + + assertEquals("chunk counts must agree for size " + size, count_iter, count_index); + long chunks_per_dim = (size + CHUNK - 1) / CHUNK; + assertEquals("chunk count for size " + size, chunks_per_dim * chunks_per_dim, count_iter); + + System.out.printf("%10d %14.3f %14.3f %10.3f%n", size, iter_ms, indx_ms, indx_ms / iter_ms); + } + } +} From 87fde45c60f3a9f4431bcdcc4af6e0ca7581f8e5 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 20 Jul 2026 20:01:25 -0400 Subject: [PATCH 02/15] Expose H5Dchunk_iter, H5Dget_num_chunks, H5Dget_chunk_info in FFM Java bindings Adds the FFM H5.java wrappers (upcall stub via the jextract-generated H5D_chunk_iter_op_t typedef class) for H5Dchunk_iter, plus H5Dget_num_chunks and H5Dget_chunk_info for by-index chunk inspection, with matching H5D_chunk_iter_cb/H5D_chunk_iter_t callback interfaces. Includes JUnit coverage and the same quick benchmark added to the JNI tree, comparing H5Dchunk_iter against a per-index H5Dget_chunk_info loop per https://github.com/JuliaIO/HDF5.jl/pull/1031#issuecomment-1407749686. --- java/hdf/hdf5lib/CMakeLists.txt | 2 + java/hdf/hdf5lib/H5.java | 129 ++++++++++++++++ .../hdf5lib/callbacks/H5D_chunk_iter_cb.java | 46 ++++++ .../hdf5lib/callbacks/H5D_chunk_iter_t.java | 24 +++ java/test/CMakeLists.txt | 1 + java/test/TestH5D.java | 52 +++++++ java/test/TestH5DChunkIterPerf.java | 144 ++++++++++++++++++ 7 files changed, 398 insertions(+) create mode 100644 java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java create mode 100644 java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java create mode 100644 java/test/TestH5DChunkIterPerf.java diff --git a/java/hdf/hdf5lib/CMakeLists.txt b/java/hdf/hdf5lib/CMakeLists.txt index ac7d34cb7ef3..5543bdeeceae 100644 --- a/java/hdf/hdf5lib/CMakeLists.txt +++ b/java/hdf/hdf5lib/CMakeLists.txt @@ -19,6 +19,8 @@ set (HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES callbacks/H5A_iterate_t.java callbacks/H5D_append_cb.java callbacks/H5D_append_t.java + callbacks/H5D_chunk_iter_cb.java + callbacks/H5D_chunk_iter_t.java callbacks/H5D_iterate_cb.java callbacks/H5D_iterate_t.java callbacks/H5E_walk_cb.java diff --git a/java/hdf/hdf5lib/H5.java b/java/hdf/hdf5lib/H5.java index df2669332f6c..db6c0798442a 100644 --- a/java/hdf/hdf5lib/H5.java +++ b/java/hdf/hdf5lib/H5.java @@ -3900,6 +3900,50 @@ public static long H5Dget_type(long dataset_id) throws HDF5LibraryException return id; } + /** + * @ingroup JH5D + * + * H5Dchunk_iter iterates over all chunks in the dataset, calling the user supplied callback with the + * details of the chunk and the supplied context op_data. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * @param op + * IN: Callback function to operate on each chunk. + * @param op_data + * IN/OUT: Pointer to any user-defined data for use by operator function. + * + * @return returns the return value of the first operator that returns a positive value, or zero if all + * chunks were processed with no operator returning non-zero. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * op is null. + **/ + public static int H5Dchunk_iter(long dataset_id, long dxpl_id, hdf.hdf5lib.callbacks.H5D_chunk_iter_cb op, + hdf.hdf5lib.callbacks.H5D_chunk_iter_t op_data) + throws HDF5LibraryException, NullPointerException + { + if (op == null) { + throw new NullPointerException("op is null"); + } + + int status = -1; + try (Arena arena = Arena.ofConfined()) { + MemorySegment op_segment = H5D_chunk_iter_op_t.allocate(op, arena); + MemorySegment op_data_segment = + Linker.nativeLinker().upcallStub(H5Dchunk_iter$handle(), H5Dchunk_iter$descriptor(), arena); + + if ((status = org.hdfgroup.javahdf5.hdf5_h.H5Dchunk_iter(dataset_id, dxpl_id, op_segment, + op_data_segment)) < 0) + h5libraryError(); + } + return status; + } + /** * @ingroup JH5D * @@ -5970,6 +6014,91 @@ public static void H5Drefresh(long dataset_id) throws HDF5LibraryException } } + /** + * @ingroup JH5D + * + * H5Dget_num_chunks retrieves the number of chunks that have a nonempty intersection with the + * selection specified by fspace_id. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param fspace_id + * IN: File dataspace selection identifier. + * + * @return the number of chunks + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public static long H5Dget_num_chunks(long dataset_id, long fspace_id) throws HDF5LibraryException + { + try (Arena arena = Arena.ofConfined()) { + MemorySegment nchunks_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_num_chunks(dataset_id, fspace_id, nchunks_segment) < 0) + h5libraryError(); + + return nchunks_segment.get(ValueLayout.JAVA_LONG, 0); + } + } + + /** + * @ingroup JH5D + * + * H5Dget_chunk_info retrieves the offset, filter mask, address, and size for the chunk specified + * by its index chk_idx within the selection specified by fspace_id. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param fspace_id + * IN: File dataspace selection identifier. + * @param chk_idx + * IN: Index of the chunk. + * @param offset + * OUT: Array of size equal to the dataset's rank; filled with the logical position of + * the chunk's first element in each dimension. + * @param filter_mask + * OUT: Array of size one; filled with the bitmask indicating the filters used when the + * chunk was written. + * @param addr + * OUT: Array of size one; filled with the chunk address in the file. + * @param size + * OUT: Array of size one; filled with the chunk size in bytes, 0 if the chunk does not + * exist. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * an output array is null. + **/ + public static void H5Dget_chunk_info(long dataset_id, long fspace_id, long chk_idx, long[] offset, + int[] filter_mask, long[] addr, long[] size) + throws HDF5LibraryException, NullPointerException + { + if (offset == null || filter_mask == null || addr == null || size == null || offset.length < 1 || + filter_mask.length < 1 || addr.length < 1 || size.length < 1) { + throw new NullPointerException("one or more arrays are null or have insufficient length"); + } + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offset_segment = arena.allocate(ValueLayout.JAVA_LONG, offset.length); + MemorySegment filter_mask_segment = arena.allocate(ValueLayout.JAVA_INT, 1); + MemorySegment addr_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + MemorySegment size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info(dataset_id, fspace_id, chk_idx, + offset_segment, filter_mask_segment, + addr_segment, size_segment) < 0) + h5libraryError(); + + for (int i = 0; i < offset.length; i++) + offset[i] = offset_segment.get(ValueLayout.JAVA_LONG, (long)i * ValueLayout.JAVA_LONG.byteSize()); + filter_mask[0] = filter_mask_segment.get(ValueLayout.JAVA_INT, 0); + addr[0] = addr_segment.get(ValueLayout.JAVA_LONG, 0); + size[0] = size_segment.get(ValueLayout.JAVA_LONG, 0); + } + } + // /////// unimplemented //////// // herr_t H5Ddebug(hid_t dset_id); // herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); diff --git a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java new file mode 100644 index 000000000000..cfca31cf427f --- /dev/null +++ b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java @@ -0,0 +1,46 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package hdf.hdf5lib.callbacks; + +import static org.hdfgroup.javahdf5.hdf5_h.*; + +import java.lang.foreign.MemorySegment; + +import org.hdfgroup.javahdf5.*; + +/** + * Information class for link callback for H5Dchunk_iter. + * + */ +public interface H5D_chunk_iter_cb extends org.hdfgroup.javahdf5.H5D_chunk_iter_op_t.Function { + /** + * @ingroup JCALLBK + * + * application callback for each chunk of a chunked dataset + * + * @param offset the logical position of the chunk's first element in units of dataset elements + * @param filter_mask bitmask indicating the filters used when the chunk was written + * @param addr the chunk address in the file, taking the user block (if any) into account + * @param size the chunk size in bytes, 0 if the chunk does not exist + * @param op_data the operator data passed in to H5Dchunk_iter + * + * @return operation status + * A. Zero causes the iterator to continue, returning zero when all + * chunks have been processed. + * B. Positive causes the iterator to immediately return that positive + * value, indicating short-circuit success. + * C. Negative causes the iterator to immediately return that value, + * indicating failure. + */ + int apply(MemorySegment offset, int filter_mask, long addr, long size, MemorySegment op_data); +} diff --git a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java new file mode 100644 index 000000000000..488640272779 --- /dev/null +++ b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java @@ -0,0 +1,24 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package hdf.hdf5lib.callbacks; + +/** + * Data class for link callback for H5Dchunk_iter. + * + */ +public interface H5D_chunk_iter_t { + /** + * public ArrayList iterdata = new ArrayList(); + * Any derived interfaces must define the single public variable as above. + */ +} diff --git a/java/test/CMakeLists.txt b/java/test/CMakeLists.txt index 318200c62eaf..5bfc9a1c9f6e 100644 --- a/java/test/CMakeLists.txt +++ b/java/test/CMakeLists.txt @@ -30,6 +30,7 @@ set (HDF5_JAVA_TEST_SOURCES TestH5D TestH5Dplist TestH5Drw + TestH5DChunkIterPerf TestH5Lparams TestH5Lbasic TestH5Lcreate diff --git a/java/test/TestH5D.java b/java/test/TestH5D.java index 05569145232b..e82d818cd733 100644 --- a/java/test/TestH5D.java +++ b/java/test/TestH5D.java @@ -34,6 +34,8 @@ import hdf.hdf5lib.H5; import hdf.hdf5lib.HDF5Constants; import hdf.hdf5lib.HDFArray; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; import hdf.hdf5lib.callbacks.H5D_iterate_cb; import hdf.hdf5lib.callbacks.H5D_iterate_t; import hdf.hdf5lib.exceptions.HDF5Exception; @@ -833,6 +835,56 @@ public void testH5Dfill() buf_data[(indx * DIM_Y) + jndx].compareTo(254) == 0); } + @Test + public void testH5Dchunk_iter() + { + final int[] chunk_count = {0}; + + class H5D_chunk_iter_data implements H5D_chunk_iter_t { + } + + class H5D_chunk_iter_callback implements H5D_chunk_iter_cb { + public int apply(MemorySegment offset, int filter_mask, long addr, long size, + MemorySegment op_data) + { + MemorySegment sized_offset = offset.reinterpret((long)RANK * ValueLayout.JAVA_LONG.byteSize()); + assertEquals("testH5Dchunk_iter: offset rank", RANK, + (int)(sized_offset.byteSize() / ValueLayout.JAVA_LONG.byteSize())); + chunk_count[0]++; + return 0; + } + } + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_iter", HDF5Constants.H5P_DEFAULT); + + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + H5D_chunk_iter_cb iter_cb = new H5D_chunk_iter_callback(); + H5D_chunk_iter_t iter_data = new H5D_chunk_iter_data(); + int op_status = -1; + try { + op_status = H5.H5Dchunk_iter(H5did, HDF5Constants.H5P_DEFAULT, iter_cb, iter_data); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dchunk_iter: " + err); + } + assertTrue("H5Dchunk_iter ", op_status == 0); + assertEquals("testH5Dchunk_iter: chunk count", 2, chunk_count[0]); + } + @Ignore public void testH5Diterate() { diff --git a/java/test/TestH5DChunkIterPerf.java b/java/test/TestH5DChunkIterPerf.java new file mode 100644 index 000000000000..d3eeaac65f72 --- /dev/null +++ b/java/test/TestH5DChunkIterPerf.java @@ -0,0 +1,144 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package test; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.lang.foreign.MemorySegment; + +import hdf.hdf5lib.H5; +import hdf.hdf5lib.HDF5Constants; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; +import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Quick benchmark comparing H5Dchunk_iter against looping over H5Dget_chunk_info by + * chunk index, following the methodology from + * https://github.com/JuliaIO/HDF5.jl/pull/1031#issuecomment-1407749686 which found that + * H5Dchunk_iter scales much better than repeated by-index lookups as the chunk count grows. + */ +public class TestH5DChunkIterPerf { + private static final String H5_FILE = "testDChunkIterPerf.h5"; + private static final int CHUNK = 16; + private static final int[] SIZES = {32, 64, 128, 256, 512, 1024}; + + long H5fid = HDF5Constants.H5I_INVALID_HID; + + private final void _deleteFile(String filename) + { + File file = new File(filename); + if (file.exists()) { + try { + file.delete(); + } + catch (SecurityException e) { + } + } + } + + @Before + public void createH5file() throws Exception + { + _deleteFile(H5_FILE); + H5fid = H5.H5Fcreate(H5_FILE, HDF5Constants.H5F_ACC_TRUNC, HDF5Constants.H5P_DEFAULT, + HDF5Constants.H5P_DEFAULT); + } + + @After + public void deleteH5file() throws Exception + { + if (H5fid >= 0) + H5.H5Fclose(H5fid); + _deleteFile(H5_FILE); + } + + private long createChunkedDataset(int size) throws Exception + { + long dcpl_id = H5.H5Pcreate(HDF5Constants.H5P_DATASET_CREATE); + H5.H5Pset_alloc_time(dcpl_id, HDF5Constants.H5D_ALLOC_TIME_EARLY); + H5.H5Pset_chunk(dcpl_id, 2, new long[] {CHUNK, CHUNK}); + + long sid = H5.H5Screate_simple(2, new long[] {size, size}, null); + long did = H5.H5Dcreate(H5fid, "dset" + size, HDF5Constants.H5T_NATIVE_UINT8, sid, + HDF5Constants.H5P_DEFAULT, dcpl_id, HDF5Constants.H5P_DEFAULT); + H5.H5Pclose(dcpl_id); + H5.H5Sclose(sid); + return did; + } + + private long countByIterate(long did) throws Exception + { + final long[] count = {0}; + class Data implements H5D_chunk_iter_t { + } + H5D_chunk_iter_cb cb = new H5D_chunk_iter_cb() { + public int apply(MemorySegment offset, int filter_mask, long addr, long size, + MemorySegment op_data) + { + count[0]++; + return 0; + } + }; + H5.H5Dchunk_iter(did, HDF5Constants.H5P_DEFAULT, cb, new Data()); + return count[0]; + } + + private long countByIndex(long did) throws Exception + { + long sid = H5.H5Dget_space(did); + long nchunks = H5.H5Dget_num_chunks(did, sid); + long[] offset = new long[2]; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; + + for (long i = 0; i < nchunks; i++) + H5.H5Dget_chunk_info(did, sid, i, offset, filter_mask, addr, size); + + H5.H5Sclose(sid); + return nchunks; + } + + @Test + public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception + { + System.out.println(); + System.out.printf("%10s %14s %14s %10s%n", "size", "iter_ms", "indx_ms", "ratio"); + + for (int size : SIZES) { + long did = createChunkedDataset(size); + + long t0 = System.nanoTime(); + long count_iter = countByIterate(did); + long t1 = System.nanoTime(); + long count_index = countByIndex(did); + long t2 = System.nanoTime(); + + H5.H5Dclose(did); + + double iter_ms = (t1 - t0) / 1.0e6; + double indx_ms = (t2 - t1) / 1.0e6; + + assertEquals("chunk counts must agree for size " + size, count_iter, count_index); + long chunks_per_dim = (size + CHUNK - 1) / CHUNK; + assertEquals("chunk count for size " + size, chunks_per_dim * chunks_per_dim, count_iter); + + System.out.printf("%10d %14.3f %14.3f %10.3f%n", size, iter_ms, indx_ms, indx_ms / iter_ms); + } + } +} From 4062dd58367a05820c1bfa91ea23c29a27daa8b2 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 00:47:59 -0400 Subject: [PATCH 03/15] Expose direct chunk I/O and inspection functions in JNI Java bindings Adds H5Dget_chunk_info_by_coord, H5Dget_chunk_storage_size, H5Dget_chunk_index_type, H5Dwrite_chunk, and H5Dread_chunk (calling the current H5Dread_chunk2 symbol) to the JNI tree, rounding out low-level raw chunk access alongside the H5Dchunk_iter/H5Dget_num_chunks/H5Dget_chunk_info added earlier this session. Also adds the five missing H5D_CHUNK_IDX_* HDF5Constants values needed to interpret H5Dget_chunk_index_type's result. H5Dread_chunk guards against a real silent-failure hazard in the underlying H5D__chunk_direct_read(): if the caller's buffer doesn't match the true on-disk chunk size, the C function still returns success but leaves the buffer untouched. Since the Java API doesn't expose the in/out buf_size parameter, the JNI glue checks it internally and throws IllegalArgumentException on a size mismatch instead of silently returning stale data. Includes JUnit coverage for each new function, including the buffer-size mismatch case. --- java/src-jni/hdf/hdf5lib/H5.java | 129 +++++++++- java/src-jni/hdf/hdf5lib/HDF5Constants.java | 20 ++ java/src-jni/jni/h5Constants.c | 25 ++ java/src-jni/jni/h5dImp.c | 256 ++++++++++++++++++++ java/src-jni/jni/h5dImp.h | 40 +++ java/src-jni/test/TestH5D.java | 144 +++++++++++ 6 files changed, 612 insertions(+), 2 deletions(-) diff --git a/java/src-jni/hdf/hdf5lib/H5.java b/java/src-jni/hdf/hdf5lib/H5.java index ca85d134cd42..fd9d05265311 100644 --- a/java/src-jni/hdf/hdf5lib/H5.java +++ b/java/src-jni/hdf/hdf5lib/H5.java @@ -4326,11 +4326,136 @@ public synchronized static native void H5Dget_chunk_info(long dataset_id, long f long[] size) throws HDF5LibraryException, NullPointerException; + /** + * @ingroup JH5D + * + * H5Dget_chunk_info_by_coord retrieves the filter mask, address, and size for the chunk in the + * dataset specified by dataset_id, using the coordinates specified by offset. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. + * @param filter_mask + * OUT: Array of size one; filled with the bitmask indicating the filters used when the + * chunk was written. + * @param addr + * OUT: Array of size one; filled with the chunk address in the file. + * @param size + * OUT: Array of size one; filled with the chunk size in bytes, 0 if the chunk does not + * exist. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * an output array is null. + **/ + public synchronized static native void H5Dget_chunk_info_by_coord(long dataset_id, long[] offset, + int[] filter_mask, long[] addr, + long[] size) + throws HDF5LibraryException, NullPointerException; + + /** + * @ingroup JH5D + * + * H5Dget_chunk_storage_size returns the size in bytes of the chunk, allocated in the file, which + * corresponds to the coordinates specified by offset. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. + * + * @return the chunk size in bytes, 0 if the chunk does not exist. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * offset is null. + **/ + public synchronized static native long H5Dget_chunk_storage_size(long dataset_id, long[] offset) + throws HDF5LibraryException, NullPointerException; + + /** + * @ingroup JH5D + * + * H5Dget_chunk_index_type queries the dataset's chunk indexing type. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * + * @return the dataset's chunk indexing type, one of the HDF5Constants.H5D_CHUNK_IDX_* values. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native int H5Dget_chunk_index_type(long dataset_id) + throws HDF5LibraryException; + + /** + * @ingroup JH5D + * + * H5Dwrite_chunk writes a raw data chunk from a buffer directly to a dataset in a file, bypassing + * the library's internal data transfer pipeline, including the filters. + * + * @param dataset_id + * IN: Identifier of the dataset to write to. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * @param filters + * IN: Mask for identifying the filters used with the chunk. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. Must specify a point on a dataset chunk boundary. + * @param buf + * IN: Buffer containing the data to be written to the chunk; its length is used as the + * chunk's data size in bytes. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * offset or buf is null. + **/ + public synchronized static native void H5Dwrite_chunk(long dataset_id, long dxpl_id, int filters, + long[] offset, byte[] buf) + throws HDF5LibraryException, NullPointerException; + + /** + * @ingroup JH5D + * + * H5Dread_chunk reads a raw data chunk directly from a dataset in a file into a buffer, bypassing + * the library's internal data transfer pipeline, including the filters. + * + * @param dataset_id + * IN: Identifier of the dataset to read from. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. Must specify a point on a dataset chunk boundary. + * @param buf + * OUT: Buffer to receive the data read from the chunk; must be exactly the size of the + * on-disk chunk, obtainable via H5Dget_chunk_info(), H5Dget_chunk_info_by_coord(), or + * H5Dget_chunk_storage_size(). + * + * @return the filter mask indicating the filters used when the chunk was written. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * offset or buf is null. + * @exception IllegalArgumentException + * buf's length does not match the chunk's actual on-disk size. + **/ + public synchronized static native int H5Dread_chunk(long dataset_id, long dxpl_id, long[] offset, + byte[] buf) + throws HDF5LibraryException, NullPointerException, IllegalArgumentException; + // /////// unimplemented //////// // herr_t H5Ddebug(hid_t dset_id); - // herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); // herr_t H5Dformat_convert(hid_t dset_id); - // herr_t H5Dget_chunk_index_type(hid_t did, H5D_chunk_index_t *idx_type); // herr_t H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, // size_t dst_buf_size, void *dst_buf, H5D_gather_func_t op, void *op_data); diff --git a/java/src-jni/hdf/hdf5lib/HDF5Constants.java b/java/src-jni/hdf/hdf5lib/HDF5Constants.java index 1c83276018d9..30e96548fc06 100644 --- a/java/src-jni/hdf/hdf5lib/HDF5Constants.java +++ b/java/src-jni/hdf/hdf5lib/HDF5Constants.java @@ -120,6 +120,16 @@ public class HDF5Constants { /** */ public static final int H5D_CHUNK_IDX_BTREE = H5D_CHUNK_IDX_BTREE(); /** */ + public static final int H5D_CHUNK_IDX_SINGLE = H5D_CHUNK_IDX_SINGLE(); + /** */ + public static final int H5D_CHUNK_IDX_NONE = H5D_CHUNK_IDX_NONE(); + /** */ + public static final int H5D_CHUNK_IDX_FARRAY = H5D_CHUNK_IDX_FARRAY(); + /** */ + public static final int H5D_CHUNK_IDX_EARRAY = H5D_CHUNK_IDX_EARRAY(); + /** */ + public static final int H5D_CHUNK_IDX_BT2 = H5D_CHUNK_IDX_BT2(); + /** */ public static final int H5D_ALLOC_TIME_DEFAULT = H5D_ALLOC_TIME_DEFAULT(); /** */ public static final int H5D_ALLOC_TIME_EARLY = H5D_ALLOC_TIME_EARLY(); @@ -1640,6 +1650,16 @@ public class HDF5Constants { private static native final int H5D_CHUNK_IDX_BTREE(); + private static native final int H5D_CHUNK_IDX_SINGLE(); + + private static native final int H5D_CHUNK_IDX_NONE(); + + private static native final int H5D_CHUNK_IDX_FARRAY(); + + private static native final int H5D_CHUNK_IDX_EARRAY(); + + private static native final int H5D_CHUNK_IDX_BT2(); + private static native final int H5D_ALLOC_TIME_DEFAULT(); private static native final int H5D_ALLOC_TIME_EARLY(); diff --git a/java/src-jni/jni/h5Constants.c b/java/src-jni/jni/h5Constants.c index 2876bcf76067..c494afe72537 100644 --- a/java/src-jni/jni/h5Constants.c +++ b/java/src-jni/jni/h5Constants.c @@ -167,6 +167,31 @@ Java_hdf_hdf5lib_HDF5Constants_H5D_1CHUNK_1IDX_1BTREE(JNIEnv *env, jclass cls) return H5D_CHUNK_IDX_BTREE; } JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_HDF5Constants_H5D_1CHUNK_1IDX_1SINGLE(JNIEnv *env, jclass cls) +{ + return H5D_CHUNK_IDX_SINGLE; +} +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_HDF5Constants_H5D_1CHUNK_1IDX_1NONE(JNIEnv *env, jclass cls) +{ + return H5D_CHUNK_IDX_NONE; +} +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_HDF5Constants_H5D_1CHUNK_1IDX_1FARRAY(JNIEnv *env, jclass cls) +{ + return H5D_CHUNK_IDX_FARRAY; +} +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_HDF5Constants_H5D_1CHUNK_1IDX_1EARRAY(JNIEnv *env, jclass cls) +{ + return H5D_CHUNK_IDX_EARRAY; +} +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_HDF5Constants_H5D_1CHUNK_1IDX_1BT2(JNIEnv *env, jclass cls) +{ + return H5D_CHUNK_IDX_BT2; +} +JNIEXPORT jint JNICALL Java_hdf_hdf5lib_HDF5Constants_H5D_1ALLOC_1TIME_1DEFAULT(JNIEnv *env, jclass cls) { return H5D_ALLOC_TIME_DEFAULT; diff --git a/java/src-jni/jni/h5dImp.c b/java/src-jni/jni/h5dImp.c index b7842ab88acb..0ab95b481c01 100644 --- a/java/src-jni/jni/h5dImp.c +++ b/java/src-jni/jni/h5dImp.c @@ -2397,6 +2397,262 @@ Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *env, jclass clss, jlong dataset_ UNPIN_LONG_ARRAY(ENVONLY, offset, offsetArray, (status < 0) ? JNI_ABORT : 0); } /* end Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info */ +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_info_by_coord + * Signature: (J[J[I[J[J)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord(JNIEnv *env, jclass clss, jlong dataset_id, + jlongArray offset, jintArray filter_mask, + jlongArray addr, jlongArray size) +{ + jboolean isCopy; + jlong *offsetArray = NULL; + jint *maskArray = NULL; + jlong *addrArray = NULL; + jlong *sizeArray = NULL; + hsize_t *offsetBuf = NULL; + hsize_t size_val = 0; + haddr_t addr_val = HADDR_UNDEF; + unsigned filter_mask_val = 0; + jsize rank; + jsize i; + herr_t status = FAIL; + + UNUSED(clss); + + if (NULL == offset) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info_by_coord: offset is NULL"); + if (NULL == filter_mask) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info_by_coord: filter_mask is NULL"); + if (NULL == addr) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info_by_coord: addr is NULL"); + if (NULL == size) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_info_by_coord: size is NULL"); + + if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, + "H5Dget_chunk_info_by_coord: offset not pinned"); + + if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dget_chunk_info_by_coord: failed to allocate offset buffer"); + + for (i = 0; i < rank; i++) + offsetBuf[i] = (hsize_t)offsetArray[i]; + + if ((status = H5Dget_chunk_info_by_coord((hid_t)dataset_id, offsetBuf, &filter_mask_val, &addr_val, + &size_val)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + PIN_INT_ARRAY(ENVONLY, filter_mask, maskArray, &isCopy, + "H5Dget_chunk_info_by_coord: filter_mask not pinned"); + maskArray[0] = (jint)filter_mask_val; + + PIN_LONG_ARRAY(ENVONLY, addr, addrArray, &isCopy, "H5Dget_chunk_info_by_coord: addr not pinned"); + addrArray[0] = (jlong)addr_val; + + PIN_LONG_ARRAY(ENVONLY, size, sizeArray, &isCopy, "H5Dget_chunk_info_by_coord: size not pinned"); + sizeArray[0] = (jlong)size_val; + +done: + if (offsetBuf) + free(offsetBuf); + if (sizeArray) + UNPIN_LONG_ARRAY(ENVONLY, size, sizeArray, (status < 0) ? JNI_ABORT : 0); + if (addrArray) + UNPIN_LONG_ARRAY(ENVONLY, addr, addrArray, (status < 0) ? JNI_ABORT : 0); + if (maskArray) + UNPIN_INT_ARRAY(ENVONLY, filter_mask, maskArray, (status < 0) ? JNI_ABORT : 0); + if (offsetArray) + UNPIN_LONG_ARRAY(ENVONLY, offset, offsetArray, JNI_ABORT); +} /* end Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_storage_size + * Signature: (J[J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Dget_1chunk_1storage_1size(JNIEnv *env, jclass clss, jlong dataset_id, + jlongArray offset) +{ + jboolean isCopy; + jlong *offsetArray = NULL; + hsize_t *offsetBuf = NULL; + hsize_t chunk_bytes = 0; + jsize rank; + jsize i; + herr_t status = FAIL; + + UNUSED(clss); + + if (NULL == offset) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dget_chunk_storage_size: offset is NULL"); + + if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, + "H5Dget_chunk_storage_size: offset not pinned"); + + if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dget_chunk_storage_size: failed to allocate offset buffer"); + + for (i = 0; i < rank; i++) + offsetBuf[i] = (hsize_t)offsetArray[i]; + + if ((status = H5Dget_chunk_storage_size((hid_t)dataset_id, offsetBuf, &chunk_bytes)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + if (offsetBuf) + free(offsetBuf); + if (offsetArray) + UNPIN_LONG_ARRAY(ENVONLY, offset, offsetArray, JNI_ABORT); + + return (jlong)chunk_bytes; +} /* end Java_hdf_hdf5lib_H5_H5Dget_1chunk_1storage_1size */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_index_type + * Signature: (J)I + */ +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Dget_1chunk_1index_1type(JNIEnv *env, jclass clss, jlong dataset_id) +{ + H5D_chunk_index_t idx_type = H5D_CHUNK_IDX_BTREE; + herr_t status = FAIL; + + UNUSED(clss); + + if ((status = H5Dget_chunk_index_type((hid_t)dataset_id, &idx_type)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return (jint)idx_type; +} /* end Java_hdf_hdf5lib_H5_H5Dget_1chunk_1index_1type */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dwrite_chunk + * Signature: (JJI[J[B)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Dwrite_1chunk(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id, + jint filters, jlongArray offset, jbyteArray buf) +{ + jboolean isCopy; + jlong *offsetArray = NULL; + jbyte *bufArray = NULL; + hsize_t *offsetBuf = NULL; + jsize rank; + jsize buf_len; + jsize i; + herr_t status = FAIL; + + UNUSED(clss); + + if (NULL == offset) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dwrite_chunk: offset is NULL"); + if (NULL == buf) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dwrite_chunk: buf is NULL"); + + if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + if ((buf_len = ENVPTR->GetArrayLength(ENVONLY, buf)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, "H5Dwrite_chunk: offset not pinned"); + + if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dwrite_chunk: failed to allocate offset buffer"); + + for (i = 0; i < rank; i++) + offsetBuf[i] = (hsize_t)offsetArray[i]; + + PIN_BYTE_ARRAY(ENVONLY, buf, bufArray, &isCopy, "H5Dwrite_chunk: buf not pinned"); + + if ((status = H5Dwrite_chunk((hid_t)dataset_id, (hid_t)dxpl_id, (uint32_t)filters, offsetBuf, + (size_t)buf_len, (const void *)bufArray)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + if (offsetBuf) + free(offsetBuf); + if (bufArray) + UNPIN_BYTE_ARRAY(ENVONLY, buf, bufArray, JNI_ABORT); + if (offsetArray) + UNPIN_LONG_ARRAY(ENVONLY, offset, offsetArray, JNI_ABORT); +} /* end Java_hdf_hdf5lib_H5_H5Dwrite_1chunk */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dread_chunk + * Signature: (JJ[J[B)I + */ +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Dread_1chunk(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id, + jlongArray offset, jbyteArray buf) +{ + jboolean isCopy; + jlong *offsetArray = NULL; + jbyte *bufArray = NULL; + hsize_t *offsetBuf = NULL; + uint32_t filters_val = 0; + size_t buf_size = 0; + jsize rank; + jsize buf_len; + jsize i; + herr_t status = FAIL; + + UNUSED(clss); + + if (NULL == offset) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dread_chunk: offset is NULL"); + if (NULL == buf) + H5_NULL_ARGUMENT_ERROR(ENVONLY, "H5Dread_chunk: buf is NULL"); + + if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + if ((buf_len = ENVPTR->GetArrayLength(ENVONLY, buf)) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); + + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, "H5Dread_chunk: offset not pinned"); + + if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dread_chunk: failed to allocate offset buffer"); + + for (i = 0; i < rank; i++) + offsetBuf[i] = (hsize_t)offsetArray[i]; + + buf_size = (size_t)buf_len; + + PIN_BYTE_ARRAY(ENVONLY, buf, bufArray, &isCopy, "H5Dread_chunk: buf not pinned"); + + if ((status = H5Dread_chunk2((hid_t)dataset_id, (hid_t)dxpl_id, offsetBuf, &filters_val, + (void *)bufArray, &buf_size)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + if ((jsize)buf_size != buf_len) { + status = FAIL; + H5_BAD_ARGUMENT_ERROR(ENVONLY, "H5Dread_chunk: buf length does not match on-disk chunk size"); + } + +done: + if (offsetBuf) + free(offsetBuf); + if (bufArray) + UNPIN_BYTE_ARRAY(ENVONLY, buf, bufArray, (status < 0) ? JNI_ABORT : 0); + if (offsetArray) + UNPIN_LONG_ARRAY(ENVONLY, offset, offsetArray, JNI_ABORT); + + return (jint)filters_val; +} /* end Java_hdf_hdf5lib_H5_H5Dread_1chunk */ + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src-jni/jni/h5dImp.h b/java/src-jni/jni/h5dImp.h index f8e32c02b814..1cb41d17f9ca 100644 --- a/java/src-jni/jni/h5dImp.h +++ b/java/src-jni/jni/h5dImp.h @@ -341,6 +341,46 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *, jclass, jlongArray, jintArray, jlongArray, jlongArray); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_info_by_coord + * Signature: (J[J[I[J[J)V + */ +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord(JNIEnv *, jclass, jlong, + jlongArray, jintArray, + jlongArray, jlongArray); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_storage_size + * Signature: (J[J)J + */ +JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1storage_1size(JNIEnv *, jclass, jlong, + jlongArray); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dget_chunk_index_type + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1index_1type(JNIEnv *, jclass, jlong); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dwrite_chunk + * Signature: (JJI[J[B)V + */ +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dwrite_1chunk(JNIEnv *, jclass, jlong, jlong, jint, + jlongArray, jbyteArray); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dread_chunk + * Signature: (JJ[J[B)I + */ +JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dread_1chunk(JNIEnv *, jclass, jlong, jlong, jlongArray, + jbyteArray); + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src-jni/test/TestH5D.java b/java/src-jni/test/TestH5D.java index bb8de406491e..99e10380f990 100644 --- a/java/src-jni/test/TestH5D.java +++ b/java/src-jni/test/TestH5D.java @@ -12,6 +12,7 @@ package test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -803,6 +804,149 @@ public int callback(long[] offset, int filter_mask, long addr, long size, H5D_ch assertEquals("testH5Dchunk_iter: chunk count", 2, chunk_count[0]); } + @Test + public void testH5Dget_chunk_info_by_coord() + { + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_info_by_coord", HDF5Constants.H5P_DEFAULT); + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + long[] offset = {0, 0}; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; + try { + H5.H5Dget_chunk_info_by_coord(H5did, offset, filter_mask, addr, size); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_info_by_coord: " + err); + } + assertTrue("testH5Dget_chunk_info_by_coord: addr", addr[0] >= 0); + assertEquals("testH5Dget_chunk_info_by_coord: size", 64, size[0]); + } + + @Test + public void testH5Dget_chunk_storage_size() + { + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_storage_size", HDF5Constants.H5P_DEFAULT); + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + long chunk_bytes = -1; + try { + chunk_bytes = H5.H5Dget_chunk_storage_size(H5did, new long[] {0, 0}); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_storage_size: " + err); + } + assertEquals("testH5Dget_chunk_storage_size", 64, chunk_bytes); + } + + @Test + public void testH5Dget_chunk_index_type() + { + _createChunkDataset(H5fid, H5dsid, "dset_chunk_index_type", HDF5Constants.H5P_DEFAULT); + + int idx_type = -1; + try { + idx_type = H5.H5Dget_chunk_index_type(H5did); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_index_type: " + err); + } + // _createChunkDataset() doesn't raise the file's format bounds (no H5Pset_libver_bounds), so + // the newer chunk-index types (FARRAY/EARRAY/BT2/SINGLE/NONE) never get selected -- that + // requires the low format bound to already be at H5O_LAYOUT_VERSION_4 or above (see the + // version-bounds gate in H5D__chunk_construct, src/H5Dchunk.c). With default/earliest bounds, + // every chunked dataset uses the original v1 B-tree index regardless of shape. + assertEquals("testH5Dget_chunk_index_type", HDF5Constants.H5D_CHUNK_IDX_BTREE, idx_type); + } + + @Test + public void testH5Dwrite_chunk_and_read_chunk() + { + _createChunkDataset(H5fid, H5dsid, "dset_write_read_chunk", HDF5Constants.H5P_DEFAULT); + + byte[] write_buf = new byte[64]; + for (int i = 0; i < write_buf.length; i++) + write_buf[i] = (byte)i; + + try { + H5.H5Dwrite_chunk(H5did, HDF5Constants.H5P_DEFAULT, 0, new long[] {0, 0}, write_buf); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dwrite_chunk: " + err); + } + + byte[] read_buf = new byte[64]; + int filters = -1; + try { + filters = H5.H5Dread_chunk(H5did, HDF5Constants.H5P_DEFAULT, new long[] {0, 0}, read_buf); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dread_chunk: " + err); + } + assertEquals("testH5Dwrite_chunk_and_read_chunk: filters", 0, filters); + assertArrayEquals("testH5Dwrite_chunk_and_read_chunk: data", write_buf, read_buf); + } + + @Test + public void testH5Dread_chunk_buffer_size_mismatch() + { + _createChunkDataset(H5fid, H5dsid, "dset_read_chunk_mismatch", HDF5Constants.H5P_DEFAULT); + + byte[] write_buf = new byte[64]; + try { + H5.H5Dwrite_chunk(H5did, HDF5Constants.H5P_DEFAULT, 0, new long[] {0, 0}, write_buf); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dwrite_chunk: " + err); + } + + byte[] undersized_buf = new byte[8]; + try { + H5.H5Dread_chunk(H5did, HDF5Constants.H5P_DEFAULT, new long[] {0, 0}, undersized_buf); + fail("H5.H5Dread_chunk: expected IllegalArgumentException for undersized buffer"); + } + catch (IllegalArgumentException expected) { + // expected + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dread_chunk: expected IllegalArgumentException, got " + err); + } + } + @Test public void testH5Diterate() { From ed41afa0fa5e1ccc34f6422ad3f064c2dcb6df66 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 00:48:11 -0400 Subject: [PATCH 04/15] Expose direct chunk I/O and inspection functions in FFM Java bindings Adds H5Dget_chunk_info_by_coord, H5Dget_chunk_storage_size, H5Dget_chunk_index_type, H5Dwrite_chunk, and H5Dread_chunk (calling the current H5Dread_chunk2 symbol) to the FFM tree, matching the JNI tree added in the same session and rounding out low-level raw chunk access alongside H5Dchunk_iter/H5Dget_num_chunks/H5Dget_chunk_info added earlier. Also adds the five missing H5D_CHUNK_IDX_* HDF5Constants values needed to interpret H5Dget_chunk_index_type's result. H5Dread_chunk guards against a real silent-failure hazard in the underlying H5D__chunk_direct_read(): if the caller's buffer doesn't match the true on-disk chunk size, the C function still returns success but leaves the buffer untouched. Since the Java API doesn't expose the in/out buf_size parameter, the wrapper checks it internally and throws IllegalArgumentException on a size mismatch instead of silently returning stale data. Includes JUnit coverage for each new function, including the buffer-size mismatch case. --- java/hdf/hdf5lib/H5.java | 223 +++++++++++++++++++++++++++- java/hdf/hdf5lib/HDF5Constants.java | 10 ++ java/test/TestH5D.java | 144 ++++++++++++++++++ 3 files changed, 375 insertions(+), 2 deletions(-) diff --git a/java/hdf/hdf5lib/H5.java b/java/hdf/hdf5lib/H5.java index db6c0798442a..e16b5fa04cfd 100644 --- a/java/hdf/hdf5lib/H5.java +++ b/java/hdf/hdf5lib/H5.java @@ -6099,11 +6099,230 @@ public static void H5Dget_chunk_info(long dataset_id, long fspace_id, long chk_i } } + /** + * @ingroup JH5D + * + * H5Dget_chunk_info_by_coord retrieves the filter mask, address, and size for the chunk in the + * dataset specified by dataset_id, using the coordinates specified by offset. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. + * @param filter_mask + * OUT: Array of size one; filled with the bitmask indicating the filters used when the + * chunk was written. + * @param addr + * OUT: Array of size one; filled with the chunk address in the file. + * @param size + * OUT: Array of size one; filled with the chunk size in bytes, 0 if the chunk does not + * exist. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * an array is null. + **/ + public static void H5Dget_chunk_info_by_coord(long dataset_id, long[] offset, int[] filter_mask, + long[] addr, long[] size) + throws HDF5LibraryException, NullPointerException + { + if (offset == null || filter_mask == null || addr == null || size == null || filter_mask.length < 1 || + addr.length < 1 || size.length < 1) { + throw new NullPointerException("one or more arrays are null or have insufficient length"); + } + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offset_segment = arena.allocateFrom(ValueLayout.JAVA_LONG, offset); + MemorySegment filter_mask_segment = arena.allocate(ValueLayout.JAVA_INT, 1); + MemorySegment addr_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + MemorySegment size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info_by_coord(dataset_id, offset_segment, + filter_mask_segment, addr_segment, + size_segment) < 0) + h5libraryError(); + + filter_mask[0] = filter_mask_segment.get(ValueLayout.JAVA_INT, 0); + addr[0] = addr_segment.get(ValueLayout.JAVA_LONG, 0); + size[0] = size_segment.get(ValueLayout.JAVA_LONG, 0); + } + } + + /** + * @ingroup JH5D + * + * H5Dget_chunk_storage_size returns the size in bytes of the chunk, allocated in the file, which + * corresponds to the coordinates specified by offset. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. + * + * @return the chunk size in bytes, 0 if the chunk does not exist. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * offset is null. + **/ + public static long H5Dget_chunk_storage_size(long dataset_id, long[] offset) + throws HDF5LibraryException, NullPointerException + { + if (offset == null) { + throw new NullPointerException("offset is null"); + } + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offset_segment = arena.allocateFrom(ValueLayout.JAVA_LONG, offset); + MemorySegment chunk_bytes_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_storage_size(dataset_id, offset_segment, + chunk_bytes_segment) < 0) + h5libraryError(); + + return chunk_bytes_segment.get(ValueLayout.JAVA_LONG, 0); + } + } + + /** + * @ingroup JH5D + * + * H5Dget_chunk_index_type queries the dataset's chunk indexing type. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * + * @return the dataset's chunk indexing type, one of the HDF5Constants.H5D_CHUNK_IDX_* values. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public static int H5Dget_chunk_index_type(long dataset_id) throws HDF5LibraryException + { + try (Arena arena = Arena.ofConfined()) { + MemorySegment idx_type_segment = arena.allocate(ValueLayout.JAVA_INT, 1); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_index_type(dataset_id, idx_type_segment) < 0) + h5libraryError(); + + return idx_type_segment.get(ValueLayout.JAVA_INT, 0); + } + } + + /** + * @ingroup JH5D + * + * H5Dwrite_chunk writes a raw data chunk from a buffer directly to a dataset in a file, bypassing + * the library's internal data transfer pipeline, including the filters. + * + * @param dataset_id + * IN: Identifier of the dataset to write to. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * @param filters + * IN: Mask for identifying the filters used with the chunk. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. Must specify a point on a dataset chunk boundary. + * @param buf + * IN: Buffer containing the data to be written to the chunk; its length is used as the + * chunk's data size in bytes. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * offset or buf is null. + **/ + public static void H5Dwrite_chunk(long dataset_id, long dxpl_id, int filters, long[] offset, byte[] buf) + throws HDF5LibraryException, NullPointerException + { + if (offset == null) { + throw new NullPointerException("offset is null"); + } + if (buf == null) { + throw new NullPointerException("buf is null"); + } + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offset_segment = arena.allocateFrom(ValueLayout.JAVA_LONG, offset); + MemorySegment buf_segment = arena.allocateFrom(ValueLayout.JAVA_BYTE, buf); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dwrite_chunk(dataset_id, dxpl_id, filters, offset_segment, + buf.length, buf_segment) < 0) + h5libraryError(); + } + } + + /** + * @ingroup JH5D + * + * H5Dread_chunk reads a raw data chunk directly from a dataset in a file into a buffer, bypassing + * the library's internal data transfer pipeline, including the filters. + * + * @param dataset_id + * IN: Identifier of the dataset to read from. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * @param offset + * IN: Array of size equal to the dataset's rank; the logical position of the chunk's + * first element in each dimension. Must specify a point on a dataset chunk boundary. + * @param buf + * OUT: Buffer to receive the data read from the chunk; must be exactly the size of the + * on-disk chunk, obtainable via H5Dget_chunk_info(), H5Dget_chunk_info_by_coord(), or + * H5Dget_chunk_storage_size(). + * + * @return the filter mask indicating the filters used when the chunk was written. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + * @exception NullPointerException + * offset or buf is null. + * @exception IllegalArgumentException + * buf's length does not match the chunk's actual on-disk size. + **/ + public static int H5Dread_chunk(long dataset_id, long dxpl_id, long[] offset, byte[] buf) + throws HDF5LibraryException, NullPointerException, IllegalArgumentException + { + if (offset == null) { + throw new NullPointerException("offset is null"); + } + if (buf == null) { + throw new NullPointerException("buf is null"); + } + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offset_segment = arena.allocateFrom(ValueLayout.JAVA_LONG, offset); + MemorySegment filters_segment = arena.allocate(ValueLayout.JAVA_INT, 1); + MemorySegment buf_segment = arena.allocate(ValueLayout.JAVA_BYTE, buf.length); + MemorySegment buf_size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + buf_size_segment.set(ValueLayout.JAVA_LONG, 0, (long)buf.length); + + if (org.hdfgroup.javahdf5.hdf5_h.H5Dread_chunk2(dataset_id, dxpl_id, offset_segment, + filters_segment, buf_segment, + buf_size_segment) < 0) + h5libraryError(); + + long actual_size = buf_size_segment.get(ValueLayout.JAVA_LONG, 0); + if (actual_size != buf.length) { + throw new IllegalArgumentException("H5Dread_chunk: buf length (" + buf.length + + ") does not match on-disk chunk size (" + actual_size + + ")"); + } + + byte[] result = buf_segment.toArray(ValueLayout.JAVA_BYTE); + System.arraycopy(result, 0, buf, 0, buf.length); + + return filters_segment.get(ValueLayout.JAVA_INT, 0); + } + } + // /////// unimplemented //////// // herr_t H5Ddebug(hid_t dset_id); - // herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); // herr_t H5Dformat_convert(hid_t dset_id); - // herr_t H5Dget_chunk_index_type(hid_t did, H5D_chunk_index_t *idx_type); // herr_t H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, // size_t dst_buf_size, void *dst_buf, H5D_gather_func_t op, void *op_data); diff --git a/java/hdf/hdf5lib/HDF5Constants.java b/java/hdf/hdf5lib/HDF5Constants.java index 6c444399fedc..fbff12b3bfba 100644 --- a/java/hdf/hdf5lib/HDF5Constants.java +++ b/java/hdf/hdf5lib/HDF5Constants.java @@ -127,6 +127,16 @@ public class HDF5Constants { /** */ public static final int H5D_CHUNK_IDX_BTREE = H5D_CHUNK_IDX_BTREE(); /** */ + public static final int H5D_CHUNK_IDX_SINGLE = H5D_CHUNK_IDX_SINGLE(); + /** */ + public static final int H5D_CHUNK_IDX_NONE = H5D_CHUNK_IDX_NONE(); + /** */ + public static final int H5D_CHUNK_IDX_FARRAY = H5D_CHUNK_IDX_FARRAY(); + /** */ + public static final int H5D_CHUNK_IDX_EARRAY = H5D_CHUNK_IDX_EARRAY(); + /** */ + public static final int H5D_CHUNK_IDX_BT2 = H5D_CHUNK_IDX_BT2(); + /** */ public static final int H5D_ALLOC_TIME_DEFAULT = H5D_ALLOC_TIME_DEFAULT(); /** */ public static final int H5D_ALLOC_TIME_EARLY = H5D_ALLOC_TIME_EARLY(); diff --git a/java/test/TestH5D.java b/java/test/TestH5D.java index e82d818cd733..5472193e5e3a 100644 --- a/java/test/TestH5D.java +++ b/java/test/TestH5D.java @@ -12,6 +12,7 @@ package test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -885,6 +886,149 @@ public int apply(MemorySegment offset, int filter_mask, long addr, long size, assertEquals("testH5Dchunk_iter: chunk count", 2, chunk_count[0]); } + @Test + public void testH5Dget_chunk_info_by_coord() + { + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_info_by_coord", HDF5Constants.H5P_DEFAULT); + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + long[] offset = {0, 0}; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; + try { + H5.H5Dget_chunk_info_by_coord(H5did, offset, filter_mask, addr, size); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_info_by_coord: " + err); + } + assertTrue("testH5Dget_chunk_info_by_coord: addr", addr[0] >= 0); + assertEquals("testH5Dget_chunk_info_by_coord: size", 64, size[0]); + } + + @Test + public void testH5Dget_chunk_storage_size() + { + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_storage_size", HDF5Constants.H5P_DEFAULT); + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + long chunk_bytes = -1; + try { + chunk_bytes = H5.H5Dget_chunk_storage_size(H5did, new long[] {0, 0}); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_storage_size: " + err); + } + assertEquals("testH5Dget_chunk_storage_size", 64, chunk_bytes); + } + + @Test + public void testH5Dget_chunk_index_type() + { + _createChunkDataset(H5fid, H5dsid, "dset_chunk_index_type", HDF5Constants.H5P_DEFAULT); + + int idx_type = -1; + try { + idx_type = H5.H5Dget_chunk_index_type(H5did); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_index_type: " + err); + } + // _createChunkDataset() doesn't raise the file's format bounds (no H5Pset_libver_bounds), so + // the newer chunk-index types (FARRAY/EARRAY/BT2/SINGLE/NONE) never get selected -- that + // requires the low format bound to already be at H5O_LAYOUT_VERSION_4 or above (see the + // version-bounds gate in H5D__chunk_construct, src/H5Dchunk.c). With default/earliest bounds, + // every chunked dataset uses the original v1 B-tree index regardless of shape. + assertEquals("testH5Dget_chunk_index_type", HDF5Constants.H5D_CHUNK_IDX_BTREE, idx_type); + } + + @Test + public void testH5Dwrite_chunk_and_read_chunk() + { + _createChunkDataset(H5fid, H5dsid, "dset_write_read_chunk", HDF5Constants.H5P_DEFAULT); + + byte[] write_buf = new byte[64]; + for (int i = 0; i < write_buf.length; i++) + write_buf[i] = (byte)i; + + try { + H5.H5Dwrite_chunk(H5did, HDF5Constants.H5P_DEFAULT, 0, new long[] {0, 0}, write_buf); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dwrite_chunk: " + err); + } + + byte[] read_buf = new byte[64]; + int filters = -1; + try { + filters = H5.H5Dread_chunk(H5did, HDF5Constants.H5P_DEFAULT, new long[] {0, 0}, read_buf); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dread_chunk: " + err); + } + assertEquals("testH5Dwrite_chunk_and_read_chunk: filters", 0, filters); + assertArrayEquals("testH5Dwrite_chunk_and_read_chunk: data", write_buf, read_buf); + } + + @Test + public void testH5Dread_chunk_buffer_size_mismatch() + { + _createChunkDataset(H5fid, H5dsid, "dset_read_chunk_mismatch", HDF5Constants.H5P_DEFAULT); + + byte[] write_buf = new byte[64]; + try { + H5.H5Dwrite_chunk(H5did, HDF5Constants.H5P_DEFAULT, 0, new long[] {0, 0}, write_buf); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dwrite_chunk: " + err); + } + + byte[] undersized_buf = new byte[8]; + try { + H5.H5Dread_chunk(H5did, HDF5Constants.H5P_DEFAULT, new long[] {0, 0}, undersized_buf); + fail("H5.H5Dread_chunk: expected IllegalArgumentException for undersized buffer"); + } + catch (IllegalArgumentException expected) { + // expected + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dread_chunk: expected IllegalArgumentException, got " + err); + } + } + @Ignore public void testH5Diterate() { From 08e130027332d6d834dadeeca9ceb929d46e644e Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 03:34:32 -0400 Subject: [PATCH 05/15] Cut per-chunk callback overhead in JNI H5Dchunk_iter, add benchmark warm-up H5D_chunk_iter_cb was doing three wasteful things on every single chunk callback instead of once per H5Dchunk_iter() call: - AttachCurrentThread/DetachCurrentThread, even though the callback runs synchronously on the same (already-attached) thread that entered Java_hdf_hdf5lib_H5_H5Dchunk_1iter - GetObjectClass + GetMethodID, a name/signature lookup repeated per chunk instead of resolved once - NewLongArray, allocating a fresh JVM heap array per chunk instead of reusing one All three are now resolved/allocated once in Java_hdf_hdf5lib_H5_H5Dchunk_1iter and threaded through the callback wrapper struct. Measured ~2.8x faster at 4096 chunks (3.7ms -> 1.3ms), cutting JNI's overhead relative to an equivalent pure-C H5Dchunk_iter call from ~18x to ~6.5x. Full TestH5D suite (38 tests) still passes. Reusing the same offset array across chunks means it's only valid for the duration of a single callback invocation now (documented in the callback's javadoc) -- this actually brings JNI's contract in line with the FFM callback's MemorySegment, which was always a transient view over native memory. Also adds a warm-up pass to TestH5DChunkIterPerf before timing, so the smallest (most overhead-sensitive) sweep entry isn't dominated by one-time JIT/class-loading cost unrelated to what's being compared. --- .../hdf5lib/callbacks/H5D_chunk_iter_cb.java | 6 +- java/src-jni/jni/h5dImp.c | 83 +++++++++++-------- java/src-jni/test/TestH5DChunkIterPerf.java | 14 +++- 3 files changed, 64 insertions(+), 39 deletions(-) diff --git a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java index 6a4e7390ae18..558ab15a9f6d 100644 --- a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java +++ b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java @@ -22,7 +22,11 @@ public interface H5D_chunk_iter_cb extends H5Callbacks { * * application callback for each chunk of a chunked dataset * - * @param offset the logical position of the chunk's first element in units of dataset elements + * @param offset the logical position of the chunk's first element in units of dataset + * elements. For performance, the same array object is reused for every chunk + * in a given H5Dchunk_iter() call; its contents are only valid for the + * duration of this callback invocation. Copy the values out if they need to + * be retained after the callback returns. * @param filter_mask bitmask indicating the filters used when the chunk was written * @param addr the chunk address in the file, taking the user block (if any) into account * @param size the chunk size in bytes, 0 if the chunk does not exist diff --git a/java/src-jni/jni/h5dImp.c b/java/src-jni/jni/h5dImp.c index 0ab95b481c01..c85ce2780a12 100644 --- a/java/src-jni/jni/h5dImp.c +++ b/java/src-jni/jni/h5dImp.c @@ -34,9 +34,14 @@ typedef struct _cb_wrapper { } cb_wrapper; typedef struct _chunk_iter_cb_wrapper { - jobject visit_callback; - jobject op_data; - unsigned rank; + jobject visit_callback; + jobject op_data; + unsigned rank; + /* Resolved once, before iteration starts, and reused for every chunk instead of being + * re-resolved/re-allocated on each callback invocation -- see H5D_chunk_iter_cb. */ + JNIEnv *env; + jmethodID mid; + jlongArray offsetArray; } chunk_iter_cb_wrapper; /********************/ @@ -2089,43 +2094,31 @@ static herr_t H5D_chunk_iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *cb_data) { chunk_iter_cb_wrapper *wrapper = (chunk_iter_cb_wrapper *)cb_data; - jlongArray offsetArray; - jmethodID mid; - jobject visit_callback = wrapper->visit_callback; - jclass cls; - JNIEnv *cbenv = NULL; - jint status = FAIL; - void *op_data = (void *)wrapper->op_data; - - if (JVMPTR->AttachCurrentThread(JVMPAR, (void **)&cbenv, NULL) < 0) { - CHECK_JNI_EXCEPTION(CBENVONLY, JNI_TRUE); - H5_JNI_FATAL_ERROR(CBENVONLY, "H5D_chunk_iter_cb: failed to attach current thread to JVM"); - } - - if (NULL == (cls = CBENVPTR->GetObjectClass(CBENVONLY, visit_callback))) - CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); - - if (NULL == (mid = CBENVPTR->GetMethodID(CBENVONLY, cls, "callback", - "([JIJJLhdf/hdf5lib/callbacks/H5D_chunk_iter_t;)I"))) - CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); + /* H5Dchunk_iter() runs synchronously on the thread that called into + * Java_hdf_hdf5lib_H5_H5Dchunk_1iter -- the JNIEnv captured there is still valid on this same + * thread, so there is no new/foreign thread to attach here (unlike a callback that might be + * invoked from a library-created worker thread). */ + JNIEnv *cbenv = wrapper->env; + jobject visit_callback = wrapper->visit_callback; + void *op_data = (void *)wrapper->op_data; + jint status = FAIL; if (NULL == offset) H5_NULL_ARGUMENT_ERROR(CBENVONLY, "H5D_chunk_iter_cb: offset is NULL"); - if (NULL == (offsetArray = CBENVPTR->NewLongArray(CBENVONLY, (jsize)wrapper->rank))) - CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); - - CBENVPTR->SetLongArrayRegion(CBENVONLY, offsetArray, 0, (jsize)wrapper->rank, (const jlong *)offset); + /* Refill the single offset array allocated once in Java_hdf_hdf5lib_H5_H5Dchunk_1iter instead of + * allocating a new one per chunk. The array (and its contents) is only valid for the duration of + * this callback invocation -- application code must copy the values out if it needs to retain + * them past the callback returning. */ + CBENVPTR->SetLongArrayRegion(CBENVONLY, wrapper->offsetArray, 0, (jsize)wrapper->rank, + (const jlong *)offset); CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); - status = CBENVPTR->CallIntMethod(CBENVONLY, visit_callback, mid, offsetArray, (jint)filter_mask, - (jlong)addr, (jlong)size, op_data); + status = CBENVPTR->CallIntMethod(CBENVONLY, visit_callback, wrapper->mid, wrapper->offsetArray, + (jint)filter_mask, (jlong)addr, (jlong)size, op_data); CHECK_JNI_EXCEPTION(CBENVONLY, JNI_FALSE); done: - if (cbenv) - JVMPTR->DetachCurrentThread(JVMPAR); - return (herr_t)status; } /* end H5D_chunk_iter_cb */ @@ -2138,11 +2131,12 @@ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dchunk_1iter(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id, jobject callback_op, jobject op_data) { - chunk_iter_cb_wrapper wrapper = {callback_op, op_data, 0}; - hid_t space_id = H5I_INVALID_HID; - bool close_space = false; + chunk_iter_cb_wrapper wrapper = {callback_op, op_data, 0, NULL, NULL, NULL}; + hid_t space_id = H5I_INVALID_HID; + jclass cls = NULL; + bool close_space = false; int ndims; - herr_t status = FAIL; + herr_t status = FAIL; UNUSED(clss); @@ -2162,11 +2156,30 @@ Java_hdf_hdf5lib_H5_H5Dchunk_1iter(JNIEnv *env, jclass clss, jlong dataset_id, j H5_LIBRARY_ERROR(ENVONLY); wrapper.rank = (unsigned)ndims; + /* Resolve the callback method and allocate the (reused) offset array once, up front, rather than + * on every chunk inside H5D_chunk_iter_cb -- GetMethodID is a name/signature lookup and + * NewLongArray is a JVM heap allocation, neither of which needs to happen per chunk. */ + wrapper.env = env; + + if (NULL == (cls = ENVPTR->GetObjectClass(ENVONLY, callback_op))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + if (NULL == (wrapper.mid = ENVPTR->GetMethodID(ENVONLY, cls, "callback", + "([JIJJLhdf/hdf5lib/callbacks/H5D_chunk_iter_t;)I"))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + if (NULL == (wrapper.offsetArray = ENVPTR->NewLongArray(ENVONLY, (jsize)wrapper.rank))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + if ((status = H5Dchunk_iter((hid_t)dataset_id, (hid_t)dxpl_id, (H5D_chunk_iter_op_t)H5D_chunk_iter_cb, (void *)&wrapper)) < 0) H5_LIBRARY_ERROR(ENVONLY); done: + if (wrapper.offsetArray) + ENVPTR->DeleteLocalRef(ENVONLY, wrapper.offsetArray); + if (cls) + ENVPTR->DeleteLocalRef(ENVONLY, cls); if (close_space) H5Sclose(space_id); diff --git a/java/src-jni/test/TestH5DChunkIterPerf.java b/java/src-jni/test/TestH5DChunkIterPerf.java index a3b26e29fe79..93b78eaaaf5b 100644 --- a/java/src-jni/test/TestH5DChunkIterPerf.java +++ b/java/src-jni/test/TestH5DChunkIterPerf.java @@ -66,14 +66,14 @@ public void deleteH5file() throws Exception _deleteFile(H5_FILE); } - private long createChunkedDataset(int size) throws Exception + private long createChunkedDataset(String name, int size) throws Exception { long dcpl_id = H5.H5Pcreate(HDF5Constants.H5P_DATASET_CREATE); H5.H5Pset_alloc_time(dcpl_id, HDF5Constants.H5D_ALLOC_TIME_EARLY); H5.H5Pset_chunk(dcpl_id, 2, new long[] {CHUNK, CHUNK}); long sid = H5.H5Screate_simple(2, new long[] {size, size}, null); - long did = H5.H5Dcreate(H5fid, "dset" + size, HDF5Constants.H5T_NATIVE_UINT8, sid, + long did = H5.H5Dcreate(H5fid, name, HDF5Constants.H5T_NATIVE_UINT8, sid, HDF5Constants.H5P_DEFAULT, dcpl_id, HDF5Constants.H5P_DEFAULT); H5.H5Pclose(dcpl_id); H5.H5Sclose(sid); @@ -115,11 +115,19 @@ private long countByIndex(long did) throws Exception @Test public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception { + // Warm up class loading/JIT/native-linkage overhead on a throwaway dataset before timing + // anything, so the smallest (and therefore most overhead-sensitive) sweep entry below isn't + // dominated by one-time costs that have nothing to do with the two approaches being compared. + long warmup_did = createChunkedDataset("warmup", SIZES[0]); + countByIterate(warmup_did); + countByIndex(warmup_did); + H5.H5Dclose(warmup_did); + System.out.println(); System.out.printf("%10s %14s %14s %10s%n", "size", "iter_ms", "indx_ms", "ratio"); for (int size : SIZES) { - long did = createChunkedDataset(size); + long did = createChunkedDataset("dset" + size, size); long t0 = System.nanoTime(); long count_iter = countByIterate(did); From d428354b8602c3f89cfee0bbee240011be9d917a Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 03:34:52 -0400 Subject: [PATCH 06/15] Add benchmark warm-up and shared-arena diagnostic to FFM chunk perf test Adds a warm-up pass to TestH5DChunkIterPerf before timing, so the smallest (most overhead-sensitive) sweep entry isn't dominated by one-time JIT/ class-loading/MethodHandle-linkage cost unrelated to what's being compared. Also adds countByIndexSharedArena(), a diagnostic variant of the by-index loop that calls the raw jextract binding directly with a single reused Arena/MemorySegment set instead of H5.H5Dget_chunk_info()'s one-Arena-per-call cost. This isolated how much of H5Dget_chunk_info's FFM overhead is Arena allocation (real at small chunk counts, ~2-5x) versus the downcall itself (dominant at large chunk counts, where shared-arena and per-call-arena times converge to within ~1-2% of each other and of the equivalent JNI/pure-C numbers) -- confirming the by-index approach's blowup is a C-library property, not a binding-layer one. Documents that H5D_chunk_iter_cb's MemorySegment offset parameter is a transient view over native memory owned by the H5Dchunk_iter call and must not be retained past the callback returning -- this was always true, just not previously stated explicitly. --- .../hdf5lib/callbacks/H5D_chunk_iter_cb.java | 6 +- java/test/TestH5DChunkIterPerf.java | 69 +++++++++++++++---- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java index cfca31cf427f..7a9d22885925 100644 --- a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java +++ b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java @@ -28,7 +28,11 @@ public interface H5D_chunk_iter_cb extends org.hdfgroup.javahdf5.H5D_chunk_iter_ * * application callback for each chunk of a chunked dataset * - * @param offset the logical position of the chunk's first element in units of dataset elements + * @param offset the logical position of the chunk's first element in units of dataset + * elements, as a view over native memory owned by the underlying H5Dchunk_iter + * call. It is only valid for the duration of this callback invocation -- read + * the values out (e.g. via offset.toArray(ValueLayout.JAVA_LONG)) if they need + * to be retained after the callback returns. * @param filter_mask bitmask indicating the filters used when the chunk was written * @param addr the chunk address in the file, taking the user block (if any) into account * @param size the chunk size in bytes, 0 if the chunk does not exist diff --git a/java/test/TestH5DChunkIterPerf.java b/java/test/TestH5DChunkIterPerf.java index d3eeaac65f72..22fe8498dd9b 100644 --- a/java/test/TestH5DChunkIterPerf.java +++ b/java/test/TestH5DChunkIterPerf.java @@ -15,7 +15,9 @@ import static org.junit.Assert.assertEquals; import java.io.File; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; import hdf.hdf5lib.H5; import hdf.hdf5lib.HDF5Constants; @@ -67,14 +69,14 @@ public void deleteH5file() throws Exception _deleteFile(H5_FILE); } - private long createChunkedDataset(int size) throws Exception + private long createChunkedDataset(String name, int size) throws Exception { long dcpl_id = H5.H5Pcreate(HDF5Constants.H5P_DATASET_CREATE); H5.H5Pset_alloc_time(dcpl_id, HDF5Constants.H5D_ALLOC_TIME_EARLY); H5.H5Pset_chunk(dcpl_id, 2, new long[] {CHUNK, CHUNK}); long sid = H5.H5Screate_simple(2, new long[] {size, size}, null); - long did = H5.H5Dcreate(H5fid, "dset" + size, HDF5Constants.H5T_NATIVE_UINT8, sid, + long did = H5.H5Dcreate(H5fid, name, HDF5Constants.H5T_NATIVE_UINT8, sid, HDF5Constants.H5P_DEFAULT, dcpl_id, HDF5Constants.H5P_DEFAULT); H5.H5Pclose(dcpl_id); H5.H5Sclose(sid); @@ -114,31 +116,74 @@ private long countByIndex(long did) throws Exception return nchunks; } + /** + * Diagnostic variant of countByIndex(): calls the raw jextract binding directly instead of + * H5.H5Dget_chunk_info(), reusing a single confined Arena (and its MemorySegments) across the + * whole loop instead of the public wrapper's one-Arena-per-call cost. Isolates how much of + * H5Dget_chunk_info's FFM overhead, relative to JNI, comes from per-call Arena allocation versus + * the downcall itself. + */ + private long countByIndexSharedArena(long did) throws Exception + { + long sid = H5.H5Dget_space(did); + long nchunks = H5.H5Dget_num_chunks(did, sid); + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offset_segment = arena.allocate(ValueLayout.JAVA_LONG, 2); + MemorySegment filter_mask_segment = arena.allocate(ValueLayout.JAVA_INT, 1); + MemorySegment addr_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + MemorySegment size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); + + for (long i = 0; i < nchunks; i++) + org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info(did, sid, i, offset_segment, + filter_mask_segment, addr_segment, + size_segment); + } + + H5.H5Sclose(sid); + return nchunks; + } + @Test public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception { + // Warm up class loading/JIT/native-linkage overhead on a throwaway dataset before timing + // anything, so the smallest (and therefore most overhead-sensitive) sweep entry below isn't + // dominated by one-time costs that have nothing to do with the two approaches being compared. + long warmup_did = createChunkedDataset("warmup", SIZES[0]); + countByIterate(warmup_did); + countByIndex(warmup_did); + countByIndexSharedArena(warmup_did); + H5.H5Dclose(warmup_did); + System.out.println(); - System.out.printf("%10s %14s %14s %10s%n", "size", "iter_ms", "indx_ms", "ratio"); + System.out.printf("%10s %14s %14s %16s %10s %16s%n", "size", "iter_ms", "indx_ms", "indx_shared_ms", + "ratio", "shared_ratio"); for (int size : SIZES) { - long did = createChunkedDataset(size); + long did = createChunkedDataset("dset" + size, size); - long t0 = System.nanoTime(); - long count_iter = countByIterate(did); - long t1 = System.nanoTime(); - long count_index = countByIndex(did); - long t2 = System.nanoTime(); + long t0 = System.nanoTime(); + long count_iter = countByIterate(did); + long t1 = System.nanoTime(); + long count_index = countByIndex(did); + long t2 = System.nanoTime(); + long count_index_shared = countByIndexSharedArena(did); + long t3 = System.nanoTime(); H5.H5Dclose(did); - double iter_ms = (t1 - t0) / 1.0e6; - double indx_ms = (t2 - t1) / 1.0e6; + double iter_ms = (t1 - t0) / 1.0e6; + double indx_ms = (t2 - t1) / 1.0e6; + double indx_shared_ms = (t3 - t2) / 1.0e6; assertEquals("chunk counts must agree for size " + size, count_iter, count_index); + assertEquals("chunk counts must agree for size " + size, count_iter, count_index_shared); long chunks_per_dim = (size + CHUNK - 1) / CHUNK; assertEquals("chunk count for size " + size, chunks_per_dim * chunks_per_dim, count_iter); - System.out.printf("%10d %14.3f %14.3f %10.3f%n", size, iter_ms, indx_ms, indx_ms / iter_ms); + System.out.printf("%10d %14.3f %14.3f %16.3f %10.3f %16.3f%n", size, iter_ms, indx_ms, + indx_shared_ms, indx_ms / iter_ms, indx_shared_ms / iter_ms); } } } From 24e9353d042e67a9cde6689fc48340cd43763945 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 05:12:48 -0400 Subject: [PATCH 07/15] Add H5Dchunk_iter_all bulk chunk enumeration to JNI Java bindings Adds H5Dchunk_iter_all(dataset_id, dxpl_id), a bulk convenience form of H5Dchunk_iter() that returns every chunk's offset, filter mask, address, and size as a single H5D_chunk_info_t instead of requiring the caller to author a callback. The JNI implementation accumulates chunk info in a plain C callback with zero JVM crossings per chunk (just memcpy into pre-sized native buffers, pre-sized exactly via H5Dget_num_chunks over the dataset's own dataspace), converting to Java arrays only once at the end. Measured ~2x faster than the already-optimized streaming H5Dchunk_iter at ~4000 chunks, and close to the pure-C baseline established earlier this session. H5D_chunk_info_t holds per-chunk fields as parallel primitive arrays rather than an array of per-chunk objects, avoiding one Java allocation per chunk on the way out. Includes JUnit coverage cross-checking the bulk result against the per-chunk H5Dget_chunk_info() results (matched by offset, since the two APIs aren't guaranteed to visit chunks in the same order), and an added column in TestH5DChunkIterPerf comparing all three chunk-enumeration strategies. --- java/src-jni/hdf/hdf5lib/CMakeLists.txt | 1 + java/src-jni/hdf/hdf5lib/H5.java | 27 ++++ .../hdf/hdf5lib/structs/H5D_chunk_info_t.java | 61 +++++++ java/src-jni/jni/h5dImp.c | 150 ++++++++++++++++++ java/src-jni/jni/h5dImp.h | 7 + java/src-jni/test/TestH5D.java | 81 ++++++++++ java/src-jni/test/TestH5DChunkIterPerf.java | 22 ++- 7 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 java/src-jni/hdf/hdf5lib/structs/H5D_chunk_info_t.java diff --git a/java/src-jni/hdf/hdf5lib/CMakeLists.txt b/java/src-jni/hdf/hdf5lib/CMakeLists.txt index c5d5f4c9c358..e5a82c916a92 100644 --- a/java/src-jni/hdf/hdf5lib/CMakeLists.txt +++ b/java/src-jni/hdf/hdf5lib/CMakeLists.txt @@ -88,6 +88,7 @@ set (HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES structs/H5_ih_info_t.java structs/H5A_info_t.java structs/H5AC_cache_config_t.java + structs/H5D_chunk_info_t.java structs/H5E_error2_t.java structs/H5F_info2_t.java structs/H5FD_ros3_fapl_t.java diff --git a/java/src-jni/hdf/hdf5lib/H5.java b/java/src-jni/hdf/hdf5lib/H5.java index fd9d05265311..2932285db3d4 100644 --- a/java/src-jni/hdf/hdf5lib/H5.java +++ b/java/src-jni/hdf/hdf5lib/H5.java @@ -49,6 +49,7 @@ import hdf.hdf5lib.exceptions.HDF5LibraryException; import hdf.hdf5lib.structs.H5AC_cache_config_t; import hdf.hdf5lib.structs.H5A_info_t; +import hdf.hdf5lib.structs.H5D_chunk_info_t; import hdf.hdf5lib.structs.H5E_error2_t; import hdf.hdf5lib.structs.H5FD_hdfs_fapl_t; import hdf.hdf5lib.structs.H5FD_ros3_fapl_t; @@ -2806,6 +2807,32 @@ public synchronized static native int H5Dchunk_iter(long dataset_id, long dxpl_i H5D_chunk_iter_t op_data) throws HDF5LibraryException, NullPointerException; + /** + * @ingroup JH5D + * + * H5Dchunk_iter_all is a bulk convenience form of H5Dchunk_iter(): rather than invoking a + * callback once per chunk, it collects every chunk's offset, filter mask, address, and size in + * a single native pass and returns them all at once, avoiding one Java call per chunk. In this + * JNI implementation, the accumulation happens entirely in native code with zero calls back into + * the JVM per chunk, converting to Java arrays only once at the end -- for large chunk counts + * this is substantially faster than H5Dchunk_iter() (roughly 2x at ~4000 chunks in local + * measurements). Use this when you want to process every chunk's metadata and don't need to stop + * iteration early; use H5Dchunk_iter() instead if you need short-circuiting or don't want to + * materialize every chunk's metadata at once. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * + * @return an H5D_chunk_info_t holding the offset, filter mask, address, and size of every chunk. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public synchronized static native H5D_chunk_info_t H5Dchunk_iter_all(long dataset_id, long dxpl_id) + throws HDF5LibraryException; + /** * @ingroup JH5D * diff --git a/java/src-jni/hdf/hdf5lib/structs/H5D_chunk_info_t.java b/java/src-jni/hdf/hdf5lib/structs/H5D_chunk_info_t.java new file mode 100644 index 000000000000..a4af6424a79c --- /dev/null +++ b/java/src-jni/hdf/hdf5lib/structs/H5D_chunk_info_t.java @@ -0,0 +1,61 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package hdf.hdf5lib.structs; + +import java.io.Serializable; + +/** + * Bulk result of H5Dchunk_iter_all(), holding information about every chunk of a chunked dataset. + * + * To avoid one Java object allocation per chunk, the per-chunk fields are stored as parallel + * primitive arrays rather than an array of per-chunk objects; use {@link #getOffset(int, int)} to + * read a single chunk's offset coordinate. + * + */ +public class H5D_chunk_info_t implements Serializable { + private static final long serialVersionUID = -8091628506238589401L; + + /** Number of dimensions (rank) of the dataset; used to interpret offset. */ + public final int rank; + /** Flattened chunk offsets: chunk i's coordinate in dimension d is offset[i * rank + d]. */ + public final long[] offset; + /** Bitmask indicating the filters used when each chunk was written; filterMask[i] for chunk i. */ + public final int[] filterMask; + /** Chunk address in the file for each chunk; addr[i] for chunk i. */ + public final long[] addr; + /** Chunk size in bytes for each chunk, 0 if the chunk does not exist; size[i] for chunk i. */ + public final long[] size; + + H5D_chunk_info_t(int rank, long[] offset, int[] filterMask, long[] addr, long[] size) + { + this.rank = rank; + this.offset = offset; + this.filterMask = filterMask; + this.addr = addr; + this.size = size; + } + + /** + * @return the number of chunks described by this object. + */ + public int getNumChunks() { return filterMask.length; } + + /** + * @param chunkIndex + * index of the chunk, between 0 and getNumChunks() - 1. + * @param dim + * dimension, between 0 and rank - 1. + * @return the logical position of the given chunk's first element in dimension dim. + */ + public long getOffset(int chunkIndex, int dim) { return offset[chunkIndex * rank + dim]; } +} diff --git a/java/src-jni/jni/h5dImp.c b/java/src-jni/jni/h5dImp.c index c85ce2780a12..0c3dcea3e5d7 100644 --- a/java/src-jni/jni/h5dImp.c +++ b/java/src-jni/jni/h5dImp.c @@ -44,6 +44,22 @@ typedef struct _chunk_iter_cb_wrapper { jlongArray offsetArray; } chunk_iter_cb_wrapper; +/* + * Native-only accumulator used by H5Dchunk_iter_all() (see Java_hdf_hdf5lib_H5_H5Dchunk_1iter_1all and + * H5D_chunk_iter_all_cb below). Unlike chunk_iter_cb_wrapper, this collects every chunk's info into + * plain native buffers during the C-level H5Dchunk_iter() traversal, with zero JNI calls per chunk; + * the buffers are only converted to Java arrays once, in bulk, after iteration completes. + */ +typedef struct _chunk_iter_all_data { + hsize_t *offsets; /* flattened, capacity * rank */ + unsigned *filter_masks; /* capacity */ + haddr_t *addrs; /* capacity */ + hsize_t *sizes; /* capacity */ + hsize_t count; + hsize_t capacity; + unsigned rank; +} chunk_iter_all_data; + /********************/ /* Local Prototypes */ /********************/ @@ -2186,6 +2202,140 @@ Java_hdf_hdf5lib_H5_H5Dchunk_1iter(JNIEnv *env, jclass clss, jlong dataset_id, j return (jint)status; } /* end Java_hdf_hdf5lib_H5_H5Dchunk_1iter */ +static herr_t +H5D_chunk_iter_all_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data) +{ + chunk_iter_all_data *data = (chunk_iter_all_data *)op_data; + + /* Defensive: should not happen, since the caller pre-sizes these buffers using + * H5Dget_num_chunks() over the same dataspace this traversal visits. If it ever does (e.g. a + * library-internal inconsistency), fail loudly rather than writing out of bounds. */ + if (data->count >= data->capacity) + return FAIL; + + memcpy(data->offsets + data->count * data->rank, offset, (size_t)data->rank * sizeof(hsize_t)); + data->filter_masks[data->count] = filter_mask; + data->addrs[data->count] = addr; + data->sizes[data->count] = size; + data->count++; + + return SUCCEED; +} /* end H5D_chunk_iter_all_cb */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dchunk_iter_all + * Signature: (JJ)Lhdf/hdf5lib/structs/H5D_chunk_info_t; + */ +JNIEXPORT jobject JNICALL +Java_hdf_hdf5lib_H5_H5Dchunk_1iter_1all(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id) +{ + chunk_iter_all_data data = {NULL, NULL, NULL, NULL, 0, 0, 0}; + hid_t space_id = H5I_INVALID_HID; + bool close_space = false; + int ndims; + hsize_t nchunks = 0; + jlongArray offsetArray = NULL; + jintArray filterMaskArray = NULL; + jlongArray addrArray = NULL; + jlongArray sizeArray = NULL; + jint *tmpInt = NULL; + jvalue args[5]; + jobject ret_obj = NULL; + herr_t status = FAIL; + hsize_t i; + + UNUSED(clss); + + if ((space_id = H5Dget_space((hid_t)dataset_id)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + close_space = true; + + if ((ndims = H5Sget_simple_extent_ndims(space_id)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + data.rank = (unsigned)ndims; + + /* Pre-size the accumulation buffers exactly using H5Dget_num_chunks() over the dataset's own + * dataspace, per that function's documented requirement (passing H5S_ALL does not currently + * work correctly), so H5D_chunk_iter_all_cb never needs to grow anything mid-iteration. */ + if (H5Dget_num_chunks((hid_t)dataset_id, space_id, &nchunks) < 0) + H5_LIBRARY_ERROR(ENVONLY); + data.capacity = nchunks; + + if (nchunks > 0) { + if (NULL == (data.offsets = (hsize_t *)malloc((size_t)nchunks * data.rank * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dchunk_iter_all: failed to allocate offsets buffer"); + if (NULL == (data.filter_masks = (unsigned *)malloc((size_t)nchunks * sizeof(unsigned)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dchunk_iter_all: failed to allocate filter_masks buffer"); + if (NULL == (data.addrs = (haddr_t *)malloc((size_t)nchunks * sizeof(haddr_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dchunk_iter_all: failed to allocate addrs buffer"); + if (NULL == (data.sizes = (hsize_t *)malloc((size_t)nchunks * sizeof(hsize_t)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dchunk_iter_all: failed to allocate sizes buffer"); + } + + if ((status = H5Dchunk_iter((hid_t)dataset_id, (hid_t)dxpl_id, + (H5D_chunk_iter_op_t)H5D_chunk_iter_all_cb, (void *)&data)) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + /* Bulk-convert the native buffers to Java arrays: one allocation and one copy per array, + * regardless of chunk count, instead of any per-chunk JNI traffic. */ + if (NULL == (offsetArray = ENVPTR->NewLongArray(ENVONLY, (jsize)(data.count * data.rank)))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + if (NULL == (filterMaskArray = ENVPTR->NewIntArray(ENVONLY, (jsize)data.count))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + if (NULL == (addrArray = ENVPTR->NewLongArray(ENVONLY, (jsize)data.count))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + if (NULL == (sizeArray = ENVPTR->NewLongArray(ENVONLY, (jsize)data.count))) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + if (data.count > 0) { + ENVPTR->SetLongArrayRegion(ENVONLY, offsetArray, 0, (jsize)(data.count * data.rank), + (const jlong *)data.offsets); + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + /* filter_masks is `unsigned`; jintArray wants jint. Same width, but reinterpret element-wise + * via a staging buffer rather than assuming the two array element types are bulk-copy + * compatible. */ + if (NULL == (tmpInt = (jint *)malloc((size_t)data.count * sizeof(jint)))) + H5_OUT_OF_MEMORY_ERROR(ENVONLY, + "H5Dchunk_iter_all: failed to allocate filter mask staging buffer"); + for (i = 0; i < data.count; i++) + tmpInt[i] = (jint)data.filter_masks[i]; + ENVPTR->SetIntArrayRegion(ENVONLY, filterMaskArray, 0, (jsize)data.count, tmpInt); + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + ENVPTR->SetLongArrayRegion(ENVONLY, addrArray, 0, (jsize)data.count, (const jlong *)data.addrs); + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + + ENVPTR->SetLongArrayRegion(ENVONLY, sizeArray, 0, (jsize)data.count, (const jlong *)data.sizes); + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + } + + args[0].i = (jint)data.rank; + args[1].l = offsetArray; + args[2].l = filterMaskArray; + args[3].l = addrArray; + args[4].l = sizeArray; + + CALL_CONSTRUCTOR(ENVONLY, "hdf/hdf5lib/structs/H5D_chunk_info_t", "(I[J[I[J[J)V", args, ret_obj); + +done: + if (tmpInt) + free(tmpInt); + if (data.offsets) + free(data.offsets); + if (data.filter_masks) + free(data.filter_masks); + if (data.addrs) + free(data.addrs); + if (data.sizes) + free(data.sizes); + if (close_space) + H5Sclose(space_id); + + return ret_obj; +} /* end Java_hdf_hdf5lib_H5_H5Dchunk_1iter_1all */ + static herr_t H5D_iterate_cb(void *elem, hid_t elem_id, unsigned ndim, const hsize_t *point, void *cb_data) { diff --git a/java/src-jni/jni/h5dImp.h b/java/src-jni/jni/h5dImp.h index 1cb41d17f9ca..0208ccefeefc 100644 --- a/java/src-jni/jni/h5dImp.h +++ b/java/src-jni/jni/h5dImp.h @@ -303,6 +303,13 @@ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dset_1extent(JNIEnv *, jclass, jlon */ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dchunk_1iter(JNIEnv *, jclass, jlong, jlong, jobject, jobject); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Dchunk_iter_all + * Signature: (JJ)Lhdf/hdf5lib/structs/H5D_chunk_info_t; + */ +JNIEXPORT jobject JNICALL Java_hdf_hdf5lib_H5_H5Dchunk_1iter_1all(JNIEnv *, jclass, jlong, jlong); + /* * Class: hdf_hdf5lib_H5 * Method: H5Diterate diff --git a/java/src-jni/test/TestH5D.java b/java/src-jni/test/TestH5D.java index 99e10380f990..f905d0e8eff2 100644 --- a/java/src-jni/test/TestH5D.java +++ b/java/src-jni/test/TestH5D.java @@ -33,6 +33,7 @@ import hdf.hdf5lib.callbacks.H5D_iterate_t; import hdf.hdf5lib.exceptions.HDF5Exception; import hdf.hdf5lib.exceptions.HDF5LibraryException; +import hdf.hdf5lib.structs.H5D_chunk_info_t; import org.junit.After; import org.junit.Before; @@ -804,6 +805,86 @@ public int callback(long[] offset, int filter_mask, long addr, long size, H5D_ch assertEquals("testH5Dchunk_iter: chunk count", 2, chunk_count[0]); } + @Test + public void testH5Dchunk_iter_all() + { + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_iter_all", HDF5Constants.H5P_DEFAULT); + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + H5D_chunk_info_t info = null; + try { + info = H5.H5Dchunk_iter_all(H5did, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dchunk_iter_all: " + err); + } + assertEquals("testH5Dchunk_iter_all: rank", RANK, info.rank); + assertEquals("testH5Dchunk_iter_all: chunk count", 2, info.getNumChunks()); + + // Cross-check against the per-chunk H5Dget_chunk_info() results (added earlier this session) + // to make sure the bulk path agrees with the already-verified by-index path. Matched by + // offset rather than by position -- the two APIs are not guaranteed to visit chunks in the + // same order. + java.util.Map byOffset = new java.util.HashMap(); + long sid = -1; + try { + sid = H5.H5Dget_space(H5did); + long nchunks_by_index = H5.H5Dget_num_chunks(H5did, sid); + assertEquals("testH5Dchunk_iter_all: H5Dget_num_chunks agrees", info.getNumChunks(), + nchunks_by_index); + + for (int i = 0; i < nchunks_by_index; i++) { + long[] offset = new long[RANK]; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; + H5.H5Dget_chunk_info(H5did, sid, i, offset, filter_mask, addr, size); + byOffset.put(java.util.Arrays.toString(offset), + new long[] {filter_mask[0], addr[0], size[0]}); + } + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_info: " + err); + } + finally { + if (sid >= 0) { + try { + H5.H5Sclose(sid); + } + catch (Exception e) { + } + } + } + + for (int i = 0; i < info.getNumChunks(); i++) { + long[] offset = new long[RANK]; + for (int d = 0; d < RANK; d++) + offset[d] = info.getOffset(i, d); + String key = java.util.Arrays.toString(offset); + long[] byIndexInfo = byOffset.get(key); + assertNotNull("testH5Dchunk_iter_all: no by-index match for offset " + key, byIndexInfo); + assertEquals("testH5Dchunk_iter_all: filterMask for offset " + key, byIndexInfo[0], + info.filterMask[i]); + assertEquals("testH5Dchunk_iter_all: addr for offset " + key, byIndexInfo[1], info.addr[i]); + assertEquals("testH5Dchunk_iter_all: size for offset " + key, byIndexInfo[2], info.size[i]); + } + } + @Test public void testH5Dget_chunk_info_by_coord() { diff --git a/java/src-jni/test/TestH5DChunkIterPerf.java b/java/src-jni/test/TestH5DChunkIterPerf.java index 93b78eaaaf5b..de0f8bd0980d 100644 --- a/java/src-jni/test/TestH5DChunkIterPerf.java +++ b/java/src-jni/test/TestH5DChunkIterPerf.java @@ -20,6 +20,7 @@ import hdf.hdf5lib.HDF5Constants; import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; +import hdf.hdf5lib.structs.H5D_chunk_info_t; import org.junit.After; import org.junit.Before; @@ -96,6 +97,12 @@ public int callback(long[] offset, int filter_mask, long addr, long size, H5D_ch return count[0]; } + private long countByIterateAll(long did) throws Exception + { + H5D_chunk_info_t info = H5.H5Dchunk_iter_all(did, HDF5Constants.H5P_DEFAULT); + return info.getNumChunks(); + } + private long countByIndex(long did) throws Exception { long sid = H5.H5Dget_space(did); @@ -120,11 +127,13 @@ public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception // dominated by one-time costs that have nothing to do with the two approaches being compared. long warmup_did = createChunkedDataset("warmup", SIZES[0]); countByIterate(warmup_did); + countByIterateAll(warmup_did); countByIndex(warmup_did); H5.H5Dclose(warmup_did); System.out.println(); - System.out.printf("%10s %14s %14s %10s%n", "size", "iter_ms", "indx_ms", "ratio"); + System.out.printf("%10s %14s %14s %14s %10s%n", "size", "iter_ms", "all_ms", "indx_ms", + "ratio_all"); for (int size : SIZES) { long did = createChunkedDataset("dset" + size, size); @@ -132,19 +141,24 @@ public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception long t0 = System.nanoTime(); long count_iter = countByIterate(did); long t1 = System.nanoTime(); - long count_index = countByIndex(did); + long count_all = countByIterateAll(did); long t2 = System.nanoTime(); + long count_index = countByIndex(did); + long t3 = System.nanoTime(); H5.H5Dclose(did); double iter_ms = (t1 - t0) / 1.0e6; - double indx_ms = (t2 - t1) / 1.0e6; + double all_ms = (t2 - t1) / 1.0e6; + double indx_ms = (t3 - t2) / 1.0e6; assertEquals("chunk counts must agree for size " + size, count_iter, count_index); + assertEquals("chunk counts must agree for size " + size, count_iter, count_all); long chunks_per_dim = (size + CHUNK - 1) / CHUNK; assertEquals("chunk count for size " + size, chunks_per_dim * chunks_per_dim, count_iter); - System.out.printf("%10d %14.3f %14.3f %10.3f%n", size, iter_ms, indx_ms, indx_ms / iter_ms); + System.out.printf("%10d %14.3f %14.3f %14.3f %10.3f%n", size, iter_ms, all_ms, indx_ms, + indx_ms / all_ms); } } } From 96d25edd9e43601b7ef266f0cf2176b3e0f7f110 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 05:13:06 -0400 Subject: [PATCH 08/15] Add H5Dchunk_iter_all bulk chunk enumeration to FFM Java bindings Adds H5Dchunk_iter_all(dataset_id, dxpl_id), matching the JNI tree added in the same session: a bulk convenience form of H5Dchunk_iter() returning every chunk's offset, filter mask, address, and size as a single H5D_chunk_info_t instead of requiring the caller to author a callback. Buffers are pre-sized exactly via H5Dget_num_chunks over the dataset's own dataspace, matching the JNI implementation's approach. Unlike the JNI implementation, this is NOT a performance optimization on FFM -- measured slower than the already-optimized streaming H5Dchunk_iter at large chunk counts, and documented as such in the method's javadoc. The C library still invokes a callback once per chunk, and in FFM that callback must be an upcall stub crossing back into the JVM on every invocation -- there is no way to give the native library a callback that runs without JVM involvement using java.lang.foreign alone, unlike JNI where the callback can be plain C. This method pays that same per-chunk upcall cost plus the extra work of copying each chunk's data into the accumulating buffers, so it's offered here purely for a simpler call site (no callback to write), not as a speedup. Includes JUnit coverage cross-checking the bulk result against the per-chunk H5Dget_chunk_info() results (matched by offset), and an added column in TestH5DChunkIterPerf comparing all chunk-enumeration strategies. --- java/hdf/hdf5lib/CMakeLists.txt | 1 + java/hdf/hdf5lib/H5.java | 103 ++++++++++++++++++ .../hdf/hdf5lib/structs/H5D_chunk_info_t.java | 61 +++++++++++ java/test/TestH5D.java | 81 ++++++++++++++ java/test/TestH5DChunkIterPerf.java | 28 +++-- 5 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 java/hdf/hdf5lib/structs/H5D_chunk_info_t.java diff --git a/java/hdf/hdf5lib/CMakeLists.txt b/java/hdf/hdf5lib/CMakeLists.txt index 5543bdeeceae..39aacc54ee1a 100644 --- a/java/hdf/hdf5lib/CMakeLists.txt +++ b/java/hdf/hdf5lib/CMakeLists.txt @@ -87,6 +87,7 @@ set (HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES structs/H5_ih_info_t.java structs/H5A_info_t.java structs/H5AC_cache_config_t.java + structs/H5D_chunk_info_t.java structs/H5E_error2_t.java structs/H5F_info2_t.java structs/H5G_info_t.java diff --git a/java/hdf/hdf5lib/H5.java b/java/hdf/hdf5lib/H5.java index e16b5fa04cfd..b7dfdee33072 100644 --- a/java/hdf/hdf5lib/H5.java +++ b/java/hdf/hdf5lib/H5.java @@ -69,6 +69,7 @@ import hdf.hdf5lib.exceptions.HDF5SymbolTableException; // import hdf.hdf5lib.structs.H5AC_cache_config_t; // import hdf.hdf5lib.structs.H5A_info_t; +import hdf.hdf5lib.structs.H5D_chunk_info_t; // import hdf.hdf5lib.structs.H5E_error2_t; // import hdf.hdf5lib.structs.H5FD_hdfs_fapl_t; import hdf.hdf5lib.structs.H5FD_ros3_fapl_t; @@ -3944,6 +3945,108 @@ public static int H5Dchunk_iter(long dataset_id, long dxpl_id, hdf.hdf5lib.callb return status; } + /** + * @ingroup JH5D + * + * H5Dchunk_iter_all is a bulk convenience form of H5Dchunk_iter(): rather than requiring the + * caller to author a callback, it collects every chunk's offset, filter mask, address, and size + * and returns them all at once as a single H5D_chunk_info_t. + * + * Note: unlike the JNI implementation of this method, this FFM implementation is NOT faster than + * H5Dchunk_iter() -- it is measurably slower at large chunk counts in local measurements. The C + * library still invokes a callback once per chunk, and in the FFM binding that callback must be + * an upcall stub that crosses back into the JVM on every invocation; there is no way to give the + * native library a callback that runs without any JVM involvement using java.lang.foreign alone. + * This method still does that same per-chunk upcall work, plus the extra cost of copying each + * chunk's data into the accumulating buffers, so it is offered purely for a simpler call site + * (no callback to write), not as a performance optimization. If per-chunk callback overhead is a + * bottleneck, prefer the JNI implementation of the Java bindings. + * + * @param dataset_id + * IN: Identifier of the dataset to query. + * @param dxpl_id + * IN: Identifier of a transfer property list. + * + * @return an H5D_chunk_info_t holding the offset, filter mask, address, and size of every chunk. + * + * @exception HDF5LibraryException + * Error from the HDF5 Library. + **/ + public static H5D_chunk_info_t H5Dchunk_iter_all(long dataset_id, long dxpl_id) + throws HDF5LibraryException + { + long space_id = H5Dget_space(dataset_id); + int rank; + long nchunks; + try { + rank = H5Sget_simple_extent_ndims(space_id); + if (rank < 0) + h5libraryError(); + nchunks = H5Dget_num_chunks(dataset_id, space_id); + } + finally { + H5Sclose(space_id); + } + + final int finalRank = rank; + final long finalNchunks = nchunks; + final long[] count = {0}; + int status; + + try (Arena arena = Arena.ofConfined()) { + MemorySegment offsetsSeg = + arena.allocate(ValueLayout.JAVA_LONG, Math.max(finalNchunks * finalRank, 1)); + MemorySegment filterMasksSeg = arena.allocate(ValueLayout.JAVA_INT, Math.max(finalNchunks, 1)); + MemorySegment addrsSeg = arena.allocate(ValueLayout.JAVA_LONG, Math.max(finalNchunks, 1)); + MemorySegment sizesSeg = arena.allocate(ValueLayout.JAVA_LONG, Math.max(finalNchunks, 1)); + + hdf.hdf5lib.callbacks.H5D_chunk_iter_cb cb = new hdf.hdf5lib.callbacks.H5D_chunk_iter_cb() { + public int apply(MemorySegment offset, int filter_mask, long addr, long size, + MemorySegment op_data) + { + /* Defensive: should not happen, since the buffers above are pre-sized using + * H5Dget_num_chunks() over the same dataspace this traversal visits. If it ever + * does, fail loudly rather than writing out of bounds. */ + if (count[0] >= finalNchunks) + return -1; + + MemorySegment sized_offset = + offset.reinterpret((long)finalRank * ValueLayout.JAVA_LONG.byteSize()); + MemorySegment.copy(sized_offset, 0, offsetsSeg, + count[0] * finalRank * ValueLayout.JAVA_LONG.byteSize(), + (long)finalRank * ValueLayout.JAVA_LONG.byteSize()); + filterMasksSeg.setAtIndex(ValueLayout.JAVA_INT, count[0], filter_mask); + addrsSeg.setAtIndex(ValueLayout.JAVA_LONG, count[0], addr); + sizesSeg.setAtIndex(ValueLayout.JAVA_LONG, count[0], size); + count[0]++; + return 0; + } + }; + + MemorySegment op_segment = H5D_chunk_iter_op_t.allocate(cb, arena); + MemorySegment op_data_segment = + Linker.nativeLinker().upcallStub(H5Dchunk_iter$handle(), H5Dchunk_iter$descriptor(), arena); + + if ((status = org.hdfgroup.javahdf5.hdf5_h.H5Dchunk_iter(dataset_id, dxpl_id, op_segment, + op_data_segment)) < 0) + h5libraryError(); + + long actualCount = count[0]; + long[] offsetArr = new long[(int)(actualCount * finalRank)]; + int[] filterMaskArr = new int[(int)actualCount]; + long[] addrArr = new long[(int)actualCount]; + long[] sizeArr = new long[(int)actualCount]; + + MemorySegment.copy(offsetsSeg, ValueLayout.JAVA_LONG, 0L, offsetArr, 0, offsetArr.length); + MemorySegment.copy(filterMasksSeg, ValueLayout.JAVA_INT, 0L, filterMaskArr, 0, + filterMaskArr.length); + MemorySegment.copy(addrsSeg, ValueLayout.JAVA_LONG, 0L, addrArr, 0, addrArr.length); + MemorySegment.copy(sizesSeg, ValueLayout.JAVA_LONG, 0L, sizeArr, 0, sizeArr.length); + + return new H5D_chunk_info_t(finalRank, offsetArr, filterMaskArr, addrArr, sizeArr); + } + } + /** * @ingroup JH5D * diff --git a/java/hdf/hdf5lib/structs/H5D_chunk_info_t.java b/java/hdf/hdf5lib/structs/H5D_chunk_info_t.java new file mode 100644 index 000000000000..4939db857816 --- /dev/null +++ b/java/hdf/hdf5lib/structs/H5D_chunk_info_t.java @@ -0,0 +1,61 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package hdf.hdf5lib.structs; + +import java.io.Serializable; + +/** + * Bulk result of H5Dchunk_iter_all(), holding information about every chunk of a chunked dataset. + * + * To avoid one Java object allocation per chunk, the per-chunk fields are stored as parallel + * primitive arrays rather than an array of per-chunk objects; use {@link #getOffset(int, int)} to + * read a single chunk's offset coordinate. + * + */ +public class H5D_chunk_info_t implements Serializable { + private static final long serialVersionUID = -8091628506238589401L; + + /** Number of dimensions (rank) of the dataset; used to interpret offset. */ + public final int rank; + /** Flattened chunk offsets: chunk i's coordinate in dimension d is offset[i * rank + d]. */ + public final long[] offset; + /** Bitmask indicating the filters used when each chunk was written; filterMask[i] for chunk i. */ + public final int[] filterMask; + /** Chunk address in the file for each chunk; addr[i] for chunk i. */ + public final long[] addr; + /** Chunk size in bytes for each chunk, 0 if the chunk does not exist; size[i] for chunk i. */ + public final long[] size; + + public H5D_chunk_info_t(int rank, long[] offset, int[] filterMask, long[] addr, long[] size) + { + this.rank = rank; + this.offset = offset; + this.filterMask = filterMask; + this.addr = addr; + this.size = size; + } + + /** + * @return the number of chunks described by this object. + */ + public int getNumChunks() { return filterMask.length; } + + /** + * @param chunkIndex + * index of the chunk, between 0 and getNumChunks() - 1. + * @param dim + * dimension, between 0 and rank - 1. + * @return the logical position of the given chunk's first element in dimension dim. + */ + public long getOffset(int chunkIndex, int dim) { return offset[chunkIndex * rank + dim]; } +} diff --git a/java/test/TestH5D.java b/java/test/TestH5D.java index 5472193e5e3a..749db3929b0b 100644 --- a/java/test/TestH5D.java +++ b/java/test/TestH5D.java @@ -41,6 +41,7 @@ import hdf.hdf5lib.callbacks.H5D_iterate_t; import hdf.hdf5lib.exceptions.HDF5Exception; import hdf.hdf5lib.exceptions.HDF5LibraryException; +import hdf.hdf5lib.structs.H5D_chunk_info_t; import org.junit.After; import org.junit.Before; @@ -886,6 +887,86 @@ public int apply(MemorySegment offset, int filter_mask, long addr, long size, assertEquals("testH5Dchunk_iter: chunk count", 2, chunk_count[0]); } + @Test + public void testH5Dchunk_iter_all() + { + int[][] write_dset_data = new int[DIM_X][DIM_Y]; + for (int indx = 0; indx < DIM_X; indx++) + for (int jndx = 0; jndx < DIM_Y; jndx++) + write_dset_data[indx][jndx] = indx * jndx - jndx; + + _createChunkDataset(H5fid, H5dsid, "dset_chunk_iter_all", HDF5Constants.H5P_DEFAULT); + + try { + H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, write_dset_data); + } + catch (Exception err) { + err.printStackTrace(); + fail("H5.H5Dwrite: " + err); + } + + H5D_chunk_info_t info = null; + try { + info = H5.H5Dchunk_iter_all(H5did, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dchunk_iter_all: " + err); + } + assertEquals("testH5Dchunk_iter_all: rank", RANK, info.rank); + assertEquals("testH5Dchunk_iter_all: chunk count", 2, info.getNumChunks()); + + // Cross-check against the per-chunk H5Dget_chunk_info() results (added earlier this session) + // to make sure the bulk path agrees with the already-verified by-index path. Matched by + // offset rather than by position -- the two APIs are not guaranteed to visit chunks in the + // same order. + java.util.Map byOffset = new java.util.HashMap(); + long sid = -1; + try { + sid = H5.H5Dget_space(H5did); + long nchunks_by_index = H5.H5Dget_num_chunks(H5did, sid); + assertEquals("testH5Dchunk_iter_all: H5Dget_num_chunks agrees", info.getNumChunks(), + nchunks_by_index); + + for (int i = 0; i < nchunks_by_index; i++) { + long[] offset = new long[RANK]; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; + H5.H5Dget_chunk_info(H5did, sid, i, offset, filter_mask, addr, size); + byOffset.put(java.util.Arrays.toString(offset), + new long[] {filter_mask[0], addr[0], size[0]}); + } + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dget_chunk_info: " + err); + } + finally { + if (sid >= 0) { + try { + H5.H5Sclose(sid); + } + catch (Exception e) { + } + } + } + + for (int i = 0; i < info.getNumChunks(); i++) { + long[] offset = new long[RANK]; + for (int d = 0; d < RANK; d++) + offset[d] = info.getOffset(i, d); + String key = java.util.Arrays.toString(offset); + long[] byIndexInfo = byOffset.get(key); + assertNotNull("testH5Dchunk_iter_all: no by-index match for offset " + key, byIndexInfo); + assertEquals("testH5Dchunk_iter_all: filterMask for offset " + key, byIndexInfo[0], + info.filterMask[i]); + assertEquals("testH5Dchunk_iter_all: addr for offset " + key, byIndexInfo[1], info.addr[i]); + assertEquals("testH5Dchunk_iter_all: size for offset " + key, byIndexInfo[2], info.size[i]); + } + } + @Test public void testH5Dget_chunk_info_by_coord() { diff --git a/java/test/TestH5DChunkIterPerf.java b/java/test/TestH5DChunkIterPerf.java index 22fe8498dd9b..b91ad7407612 100644 --- a/java/test/TestH5DChunkIterPerf.java +++ b/java/test/TestH5DChunkIterPerf.java @@ -23,6 +23,7 @@ import hdf.hdf5lib.HDF5Constants; import hdf.hdf5lib.callbacks.H5D_chunk_iter_cb; import hdf.hdf5lib.callbacks.H5D_chunk_iter_t; +import hdf.hdf5lib.structs.H5D_chunk_info_t; import org.junit.After; import org.junit.Before; @@ -100,6 +101,12 @@ public int apply(MemorySegment offset, int filter_mask, long addr, long size, return count[0]; } + private long countByIterateAll(long did) throws Exception + { + H5D_chunk_info_t info = H5.H5Dchunk_iter_all(did, HDF5Constants.H5P_DEFAULT); + return info.getNumChunks(); + } + private long countByIndex(long did) throws Exception { long sid = H5.H5Dget_space(did); @@ -152,13 +159,14 @@ public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception // dominated by one-time costs that have nothing to do with the two approaches being compared. long warmup_did = createChunkedDataset("warmup", SIZES[0]); countByIterate(warmup_did); + countByIterateAll(warmup_did); countByIndex(warmup_did); countByIndexSharedArena(warmup_did); H5.H5Dclose(warmup_did); System.out.println(); - System.out.printf("%10s %14s %14s %16s %10s %16s%n", "size", "iter_ms", "indx_ms", "indx_shared_ms", - "ratio", "shared_ratio"); + System.out.printf("%10s %14s %14s %14s %16s %10s %16s%n", "size", "iter_ms", "all_ms", "indx_ms", + "indx_shared_ms", "ratio_all", "shared_ratio"); for (int size : SIZES) { long did = createChunkedDataset("dset" + size, size); @@ -166,24 +174,28 @@ public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception long t0 = System.nanoTime(); long count_iter = countByIterate(did); long t1 = System.nanoTime(); - long count_index = countByIndex(did); + long count_all = countByIterateAll(did); long t2 = System.nanoTime(); - long count_index_shared = countByIndexSharedArena(did); + long count_index = countByIndex(did); long t3 = System.nanoTime(); + long count_index_shared = countByIndexSharedArena(did); + long t4 = System.nanoTime(); H5.H5Dclose(did); double iter_ms = (t1 - t0) / 1.0e6; - double indx_ms = (t2 - t1) / 1.0e6; - double indx_shared_ms = (t3 - t2) / 1.0e6; + double all_ms = (t2 - t1) / 1.0e6; + double indx_ms = (t3 - t2) / 1.0e6; + double indx_shared_ms = (t4 - t3) / 1.0e6; assertEquals("chunk counts must agree for size " + size, count_iter, count_index); + assertEquals("chunk counts must agree for size " + size, count_iter, count_all); assertEquals("chunk counts must agree for size " + size, count_iter, count_index_shared); long chunks_per_dim = (size + CHUNK - 1) / CHUNK; assertEquals("chunk count for size " + size, chunks_per_dim * chunks_per_dim, count_iter); - System.out.printf("%10d %14.3f %14.3f %16.3f %10.3f %16.3f%n", size, iter_ms, indx_ms, - indx_shared_ms, indx_ms / iter_ms, indx_shared_ms / iter_ms); + System.out.printf("%10d %14.3f %14.3f %14.3f %16.3f %10.3f %16.3f%n", size, iter_ms, all_ms, + indx_ms, indx_shared_ms, indx_ms / all_ms, indx_shared_ms / iter_ms); } } } From bd1824544c8d6830bd78605ac37002dad03f30b1 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 13:04:11 -0400 Subject: [PATCH 09/15] Apply clang-format to JNI chunk-function changes CI's Formatting Check (clang-format 17) flagged alignment/wrapping differences in the chunk-related additions from this branch. Ran clang-format 17 (matching CI exactly, via pixi) over the changed files and committed the result -- purely whitespace, no semantic changes (verified by reviewing every diff and rebuilding + rerunning the full TestH5D suite, which still passes 39/39). --- java/src-jni/hdf/hdf5lib/H5.java | 5 +- java/src-jni/jni/h5dImp.c | 70 ++++++++++----------- java/src-jni/jni/h5dImp.h | 15 ++--- java/src-jni/test/TestH5D.java | 42 ++++++------- java/src-jni/test/TestH5DChunkIterPerf.java | 13 ++-- 5 files changed, 69 insertions(+), 76 deletions(-) diff --git a/java/src-jni/hdf/hdf5lib/H5.java b/java/src-jni/hdf/hdf5lib/H5.java index 2932285db3d4..19d22bfaa0f3 100644 --- a/java/src-jni/hdf/hdf5lib/H5.java +++ b/java/src-jni/hdf/hdf5lib/H5.java @@ -4378,9 +4378,8 @@ public synchronized static native void H5Dget_chunk_info(long dataset_id, long f * @exception NullPointerException * an output array is null. **/ - public synchronized static native void H5Dget_chunk_info_by_coord(long dataset_id, long[] offset, - int[] filter_mask, long[] addr, - long[] size) + public synchronized static native void + H5Dget_chunk_info_by_coord(long dataset_id, long[] offset, int[] filter_mask, long[] addr, long[] size) throws HDF5LibraryException, NullPointerException; /** diff --git a/java/src-jni/jni/h5dImp.c b/java/src-jni/jni/h5dImp.c index 0c3dcea3e5d7..cbba28e89e51 100644 --- a/java/src-jni/jni/h5dImp.c +++ b/java/src-jni/jni/h5dImp.c @@ -34,9 +34,9 @@ typedef struct _cb_wrapper { } cb_wrapper; typedef struct _chunk_iter_cb_wrapper { - jobject visit_callback; - jobject op_data; - unsigned rank; + jobject visit_callback; + jobject op_data; + unsigned rank; /* Resolved once, before iteration starts, and reused for every chunk instead of being * re-resolved/re-allocated on each callback invocation -- see H5D_chunk_iter_cb. */ JNIEnv *env; @@ -51,13 +51,13 @@ typedef struct _chunk_iter_cb_wrapper { * the buffers are only converted to Java arrays once, in bulk, after iteration completes. */ typedef struct _chunk_iter_all_data { - hsize_t *offsets; /* flattened, capacity * rank */ + hsize_t *offsets; /* flattened, capacity * rank */ unsigned *filter_masks; /* capacity */ - haddr_t *addrs; /* capacity */ - hsize_t *sizes; /* capacity */ - hsize_t count; - hsize_t capacity; - unsigned rank; + haddr_t *addrs; /* capacity */ + hsize_t *sizes; /* capacity */ + hsize_t count; + hsize_t capacity; + unsigned rank; } chunk_iter_all_data; /********************/ @@ -2114,10 +2114,10 @@ H5D_chunk_iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsi * Java_hdf_hdf5lib_H5_H5Dchunk_1iter -- the JNIEnv captured there is still valid on this same * thread, so there is no new/foreign thread to attach here (unlike a callback that might be * invoked from a library-created worker thread). */ - JNIEnv *cbenv = wrapper->env; - jobject visit_callback = wrapper->visit_callback; - void *op_data = (void *)wrapper->op_data; - jint status = FAIL; + JNIEnv *cbenv = wrapper->env; + jobject visit_callback = wrapper->visit_callback; + void *op_data = (void *)wrapper->op_data; + jint status = FAIL; if (NULL == offset) H5_NULL_ARGUMENT_ERROR(CBENVONLY, "H5D_chunk_iter_cb: offset is NULL"); @@ -2230,9 +2230,9 @@ H5D_chunk_iter_all_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, JNIEXPORT jobject JNICALL Java_hdf_hdf5lib_H5_H5Dchunk_1iter_1all(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id) { - chunk_iter_all_data data = {NULL, NULL, NULL, NULL, 0, 0, 0}; - hid_t space_id = H5I_INVALID_HID; - bool close_space = false; + chunk_iter_all_data data = {NULL, NULL, NULL, NULL, 0, 0, 0}; + hid_t space_id = H5I_INVALID_HID; + bool close_space = false; int ndims; hsize_t nchunks = 0; jlongArray offsetArray = NULL; @@ -2273,8 +2273,8 @@ Java_hdf_hdf5lib_H5_H5Dchunk_1iter_1all(JNIEnv *env, jclass clss, jlong dataset_ H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dchunk_iter_all: failed to allocate sizes buffer"); } - if ((status = H5Dchunk_iter((hid_t)dataset_id, (hid_t)dxpl_id, - (H5D_chunk_iter_op_t)H5D_chunk_iter_all_cb, (void *)&data)) < 0) + if ((status = H5Dchunk_iter((hid_t)dataset_id, (hid_t)dxpl_id, (H5D_chunk_iter_op_t)H5D_chunk_iter_all_cb, + (void *)&data)) < 0) H5_LIBRARY_ERROR(ENVONLY); /* Bulk-convert the native buffers to Java arrays: one allocation and one copy per array, @@ -2501,13 +2501,13 @@ Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *env, jclass clss, jlong dataset_ jlongArray addr, jlongArray size) { jboolean isCopy; - jlong *offsetArray = NULL; - jint *maskArray = NULL; - jlong *addrArray = NULL; - jlong *sizeArray = NULL; - hsize_t *offsetBuf = NULL; - hsize_t size_val = 0; - haddr_t addr_val = HADDR_UNDEF; + jlong *offsetArray = NULL; + jint *maskArray = NULL; + jlong *addrArray = NULL; + jlong *sizeArray = NULL; + hsize_t *offsetBuf = NULL; + hsize_t size_val = 0; + haddr_t addr_val = HADDR_UNDEF; unsigned filter_mask_val = 0; jsize rank; jsize i; @@ -2567,8 +2567,8 @@ Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *env, jclass clss, jlong dataset_ */ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord(JNIEnv *env, jclass clss, jlong dataset_id, - jlongArray offset, jintArray filter_mask, - jlongArray addr, jlongArray size) + jlongArray offset, jintArray filter_mask, jlongArray addr, + jlongArray size) { jboolean isCopy; jlong *offsetArray = NULL; @@ -2597,8 +2597,7 @@ Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord(JNIEnv *env, jclass clss, jlo if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); - PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, - "H5Dget_chunk_info_by_coord: offset not pinned"); + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, "H5Dget_chunk_info_by_coord: offset not pinned"); if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dget_chunk_info_by_coord: failed to allocate offset buffer"); @@ -2658,8 +2657,7 @@ Java_hdf_hdf5lib_H5_H5Dget_1chunk_1storage_1size(JNIEnv *env, jclass clss, jlong if ((rank = ENVPTR->GetArrayLength(ENVONLY, offset)) < 0) CHECK_JNI_EXCEPTION(ENVONLY, JNI_TRUE); - PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, - "H5Dget_chunk_storage_size: offset not pinned"); + PIN_LONG_ARRAY(ENVONLY, offset, offsetArray, &isCopy, "H5Dget_chunk_storage_size: offset not pinned"); if (NULL == (offsetBuf = (hsize_t *)malloc((size_t)rank * sizeof(hsize_t)))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "H5Dget_chunk_storage_size: failed to allocate offset buffer"); @@ -2688,7 +2686,7 @@ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1index_1type(JNIEnv *env, jclass clss, jlong dataset_id) { H5D_chunk_index_t idx_type = H5D_CHUNK_IDX_BTREE; - herr_t status = FAIL; + herr_t status = FAIL; UNUSED(clss); @@ -2705,8 +2703,8 @@ Java_hdf_hdf5lib_H5_H5Dget_1chunk_1index_1type(JNIEnv *env, jclass clss, jlong d * Signature: (JJI[J[B)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Dwrite_1chunk(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id, - jint filters, jlongArray offset, jbyteArray buf) +Java_hdf_hdf5lib_H5_H5Dwrite_1chunk(JNIEnv *env, jclass clss, jlong dataset_id, jlong dxpl_id, jint filters, + jlongArray offset, jbyteArray buf) { jboolean isCopy; jlong *offsetArray = NULL; @@ -2796,8 +2794,8 @@ Java_hdf_hdf5lib_H5_H5Dread_1chunk(JNIEnv *env, jclass clss, jlong dataset_id, j PIN_BYTE_ARRAY(ENVONLY, buf, bufArray, &isCopy, "H5Dread_chunk: buf not pinned"); - if ((status = H5Dread_chunk2((hid_t)dataset_id, (hid_t)dxpl_id, offsetBuf, &filters_val, - (void *)bufArray, &buf_size)) < 0) + if ((status = H5Dread_chunk2((hid_t)dataset_id, (hid_t)dxpl_id, offsetBuf, &filters_val, (void *)bufArray, + &buf_size)) < 0) H5_LIBRARY_ERROR(ENVONLY); if ((jsize)buf_size != buf_len) { diff --git a/java/src-jni/jni/h5dImp.h b/java/src-jni/jni/h5dImp.h index 0208ccefeefc..7b222e2c3e24 100644 --- a/java/src-jni/jni/h5dImp.h +++ b/java/src-jni/jni/h5dImp.h @@ -345,25 +345,22 @@ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Dget_1num_1chunks(JNIEnv *, jclass * Signature: (JJJ[J[I[J[J)V */ JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info(JNIEnv *, jclass, jlong, jlong, jlong, - jlongArray, jintArray, jlongArray, - jlongArray); + jlongArray, jintArray, jlongArray, jlongArray); /* * Class: hdf_hdf5lib_H5 * Method: H5Dget_chunk_info_by_coord * Signature: (J[J[I[J[J)V */ -JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord(JNIEnv *, jclass, jlong, - jlongArray, jintArray, - jlongArray, jlongArray); +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1info_1by_1coord(JNIEnv *, jclass, jlong, jlongArray, + jintArray, jlongArray, jlongArray); /* * Class: hdf_hdf5lib_H5 * Method: H5Dget_chunk_storage_size * Signature: (J[J)J */ -JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1storage_1size(JNIEnv *, jclass, jlong, - jlongArray); +JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1storage_1size(JNIEnv *, jclass, jlong, jlongArray); /* * Class: hdf_hdf5lib_H5 @@ -377,8 +374,8 @@ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Dget_1chunk_1index_1type(JNIEnv *, * Method: H5Dwrite_chunk * Signature: (JJI[J[B)V */ -JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dwrite_1chunk(JNIEnv *, jclass, jlong, jlong, jint, - jlongArray, jbyteArray); +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Dwrite_1chunk(JNIEnv *, jclass, jlong, jlong, jint, jlongArray, + jbyteArray); /* * Class: hdf_hdf5lib_H5 diff --git a/java/src-jni/test/TestH5D.java b/java/src-jni/test/TestH5D.java index f905d0e8eff2..f03b37924dd5 100644 --- a/java/src-jni/test/TestH5D.java +++ b/java/src-jni/test/TestH5D.java @@ -763,11 +763,11 @@ public void testH5Dchunk_iter() { final int[] chunk_count = {0}; - class H5D_chunk_iter_data implements H5D_chunk_iter_t { - } + class H5D_chunk_iter_data implements H5D_chunk_iter_t {} class H5D_chunk_iter_callback implements H5D_chunk_iter_cb { - public int callback(long[] offset, int filter_mask, long addr, long size, H5D_chunk_iter_t op_data) + public int callback(long[] offset, int filter_mask, long addr, long size, + H5D_chunk_iter_t op_data) { assertEquals("testH5Dchunk_iter: offset rank", RANK, offset.length); chunk_count[0]++; @@ -784,16 +784,16 @@ public int callback(long[] offset, int filter_mask, long addr, long size, H5D_ch try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); fail("H5.H5Dwrite: " + err); } - H5D_chunk_iter_cb iter_cb = new H5D_chunk_iter_callback(); - H5D_chunk_iter_t iter_data = new H5D_chunk_iter_data(); - int op_status = -1; + H5D_chunk_iter_cb iter_cb = new H5D_chunk_iter_callback(); + H5D_chunk_iter_t iter_data = new H5D_chunk_iter_data(); + int op_status = -1; try { op_status = H5.H5Dchunk_iter(H5did, HDF5Constants.H5P_DEFAULT, iter_cb, iter_data); } @@ -817,7 +817,7 @@ public void testH5Dchunk_iter_all() try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); @@ -840,21 +840,21 @@ public void testH5Dchunk_iter_all() // offset rather than by position -- the two APIs are not guaranteed to visit chunks in the // same order. java.util.Map byOffset = new java.util.HashMap(); - long sid = -1; + long sid = -1; try { - sid = H5.H5Dget_space(H5did); + sid = H5.H5Dget_space(H5did); long nchunks_by_index = H5.H5Dget_num_chunks(H5did, sid); assertEquals("testH5Dchunk_iter_all: H5Dget_num_chunks agrees", info.getNumChunks(), - nchunks_by_index); + nchunks_by_index); for (int i = 0; i < nchunks_by_index; i++) { - long[] offset = new long[RANK]; - int[] filter_mask = new int[1]; - long[] addr = new long[1]; - long[] size = new long[1]; + long[] offset = new long[RANK]; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; H5.H5Dget_chunk_info(H5did, sid, i, offset, filter_mask, addr, size); byOffset.put(java.util.Arrays.toString(offset), - new long[] {filter_mask[0], addr[0], size[0]}); + new long[] {filter_mask[0], addr[0], size[0]}); } } catch (Throwable err) { @@ -875,11 +875,11 @@ public void testH5Dchunk_iter_all() long[] offset = new long[RANK]; for (int d = 0; d < RANK; d++) offset[d] = info.getOffset(i, d); - String key = java.util.Arrays.toString(offset); + String key = java.util.Arrays.toString(offset); long[] byIndexInfo = byOffset.get(key); assertNotNull("testH5Dchunk_iter_all: no by-index match for offset " + key, byIndexInfo); assertEquals("testH5Dchunk_iter_all: filterMask for offset " + key, byIndexInfo[0], - info.filterMask[i]); + info.filterMask[i]); assertEquals("testH5Dchunk_iter_all: addr for offset " + key, byIndexInfo[1], info.addr[i]); assertEquals("testH5Dchunk_iter_all: size for offset " + key, byIndexInfo[2], info.size[i]); } @@ -897,7 +897,7 @@ public void testH5Dget_chunk_info_by_coord() try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); @@ -931,7 +931,7 @@ public void testH5Dget_chunk_storage_size() try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); @@ -988,7 +988,7 @@ public void testH5Dwrite_chunk_and_read_chunk() } byte[] read_buf = new byte[64]; - int filters = -1; + int filters = -1; try { filters = H5.H5Dread_chunk(H5did, HDF5Constants.H5P_DEFAULT, new long[] {0, 0}, read_buf); } diff --git a/java/src-jni/test/TestH5DChunkIterPerf.java b/java/src-jni/test/TestH5DChunkIterPerf.java index de0f8bd0980d..2d86519f63f4 100644 --- a/java/src-jni/test/TestH5DChunkIterPerf.java +++ b/java/src-jni/test/TestH5DChunkIterPerf.java @@ -74,8 +74,8 @@ private long createChunkedDataset(String name, int size) throws Exception H5.H5Pset_chunk(dcpl_id, 2, new long[] {CHUNK, CHUNK}); long sid = H5.H5Screate_simple(2, new long[] {size, size}, null); - long did = H5.H5Dcreate(H5fid, name, HDF5Constants.H5T_NATIVE_UINT8, sid, - HDF5Constants.H5P_DEFAULT, dcpl_id, HDF5Constants.H5P_DEFAULT); + long did = H5.H5Dcreate(H5fid, name, HDF5Constants.H5T_NATIVE_UINT8, sid, HDF5Constants.H5P_DEFAULT, + dcpl_id, HDF5Constants.H5P_DEFAULT); H5.H5Pclose(dcpl_id); H5.H5Sclose(sid); return did; @@ -84,10 +84,10 @@ private long createChunkedDataset(String name, int size) throws Exception private long countByIterate(long did) throws Exception { final long[] count = {0}; - class Data implements H5D_chunk_iter_t { - } + class Data implements H5D_chunk_iter_t {} H5D_chunk_iter_cb cb = new H5D_chunk_iter_cb() { - public int callback(long[] offset, int filter_mask, long addr, long size, H5D_chunk_iter_t op_data) + public int callback(long[] offset, int filter_mask, long addr, long size, + H5D_chunk_iter_t op_data) { count[0]++; return 0; @@ -132,8 +132,7 @@ public void testH5Dchunk_iter_vs_get_chunk_info_perf() throws Exception H5.H5Dclose(warmup_did); System.out.println(); - System.out.printf("%10s %14s %14s %14s %10s%n", "size", "iter_ms", "all_ms", "indx_ms", - "ratio_all"); + System.out.printf("%10s %14s %14s %14s %10s%n", "size", "iter_ms", "all_ms", "indx_ms", "ratio_all"); for (int size : SIZES) { long did = createChunkedDataset("dset" + size, size); From 991e37c84e1f314c46f5458de7e7e24400f1988e Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 13:04:32 -0400 Subject: [PATCH 10/15] Apply clang-format to FFM chunk-function changes CI's Formatting Check (clang-format 17) flagged alignment/wrapping differences in the chunk-related additions from this branch. Ran clang-format 17 (matching CI exactly, via pixi) over the changed files and committed the result -- purely whitespace, no semantic changes (verified by reviewing every diff; JNI-tree equivalent rebuilt and retested to confirm the formatter didn't alter behavior). --- java/hdf/hdf5lib/H5.java | 29 +++++++++---------- java/test/TestH5D.java | 44 ++++++++++++++--------------- java/test/TestH5DChunkIterPerf.java | 12 ++++---- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/java/hdf/hdf5lib/H5.java b/java/hdf/hdf5lib/H5.java index b7dfdee33072..e28cc9458f09 100644 --- a/java/hdf/hdf5lib/H5.java +++ b/java/hdf/hdf5lib/H5.java @@ -3976,7 +3976,7 @@ public static H5D_chunk_info_t H5Dchunk_iter_all(long dataset_id, long dxpl_id) throws HDF5LibraryException { long space_id = H5Dget_space(dataset_id); - int rank; + int rank; long nchunks; try { rank = H5Sget_simple_extent_ndims(space_id); @@ -3988,10 +3988,10 @@ public static H5D_chunk_info_t H5Dchunk_iter_all(long dataset_id, long dxpl_id) H5Sclose(space_id); } - final int finalRank = rank; - final long finalNchunks = nchunks; - final long[] count = {0}; - int status; + final int finalRank = rank; + final long finalNchunks = nchunks; + final long[] count = {0}; + int status; try (Arena arena = Arena.ofConfined()) { MemorySegment offsetsSeg = @@ -6189,13 +6189,14 @@ public static void H5Dget_chunk_info(long dataset_id, long fspace_id, long chk_i MemorySegment addr_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); MemorySegment size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); - if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info(dataset_id, fspace_id, chk_idx, - offset_segment, filter_mask_segment, - addr_segment, size_segment) < 0) + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info(dataset_id, fspace_id, chk_idx, offset_segment, + filter_mask_segment, addr_segment, + size_segment) < 0) h5libraryError(); for (int i = 0; i < offset.length; i++) - offset[i] = offset_segment.get(ValueLayout.JAVA_LONG, (long)i * ValueLayout.JAVA_LONG.byteSize()); + offset[i] = + offset_segment.get(ValueLayout.JAVA_LONG, (long)i * ValueLayout.JAVA_LONG.byteSize()); filter_mask[0] = filter_mask_segment.get(ValueLayout.JAVA_INT, 0); addr[0] = addr_segment.get(ValueLayout.JAVA_LONG, 0); size[0] = size_segment.get(ValueLayout.JAVA_LONG, 0); @@ -6242,9 +6243,8 @@ public static void H5Dget_chunk_info_by_coord(long dataset_id, long[] offset, in MemorySegment addr_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); MemorySegment size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); - if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info_by_coord(dataset_id, offset_segment, - filter_mask_segment, addr_segment, - size_segment) < 0) + if (org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info_by_coord( + dataset_id, offset_segment, filter_mask_segment, addr_segment, size_segment) < 0) h5libraryError(); filter_mask[0] = filter_mask_segment.get(ValueLayout.JAVA_INT, 0); @@ -6404,9 +6404,8 @@ public static int H5Dread_chunk(long dataset_id, long dxpl_id, long[] offset, by MemorySegment buf_size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); buf_size_segment.set(ValueLayout.JAVA_LONG, 0, (long)buf.length); - if (org.hdfgroup.javahdf5.hdf5_h.H5Dread_chunk2(dataset_id, dxpl_id, offset_segment, - filters_segment, buf_segment, - buf_size_segment) < 0) + if (org.hdfgroup.javahdf5.hdf5_h.H5Dread_chunk2( + dataset_id, dxpl_id, offset_segment, filters_segment, buf_segment, buf_size_segment) < 0) h5libraryError(); long actual_size = buf_size_segment.get(ValueLayout.JAVA_LONG, 0); diff --git a/java/test/TestH5D.java b/java/test/TestH5D.java index 749db3929b0b..9498de59432e 100644 --- a/java/test/TestH5D.java +++ b/java/test/TestH5D.java @@ -842,16 +842,16 @@ public void testH5Dchunk_iter() { final int[] chunk_count = {0}; - class H5D_chunk_iter_data implements H5D_chunk_iter_t { - } + class H5D_chunk_iter_data implements H5D_chunk_iter_t {} class H5D_chunk_iter_callback implements H5D_chunk_iter_cb { public int apply(MemorySegment offset, int filter_mask, long addr, long size, MemorySegment op_data) { - MemorySegment sized_offset = offset.reinterpret((long)RANK * ValueLayout.JAVA_LONG.byteSize()); + MemorySegment sized_offset = + offset.reinterpret((long)RANK * ValueLayout.JAVA_LONG.byteSize()); assertEquals("testH5Dchunk_iter: offset rank", RANK, - (int)(sized_offset.byteSize() / ValueLayout.JAVA_LONG.byteSize())); + (int)(sized_offset.byteSize() / ValueLayout.JAVA_LONG.byteSize())); chunk_count[0]++; return 0; } @@ -866,16 +866,16 @@ public int apply(MemorySegment offset, int filter_mask, long addr, long size, try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); fail("H5.H5Dwrite: " + err); } - H5D_chunk_iter_cb iter_cb = new H5D_chunk_iter_callback(); - H5D_chunk_iter_t iter_data = new H5D_chunk_iter_data(); - int op_status = -1; + H5D_chunk_iter_cb iter_cb = new H5D_chunk_iter_callback(); + H5D_chunk_iter_t iter_data = new H5D_chunk_iter_data(); + int op_status = -1; try { op_status = H5.H5Dchunk_iter(H5did, HDF5Constants.H5P_DEFAULT, iter_cb, iter_data); } @@ -899,7 +899,7 @@ public void testH5Dchunk_iter_all() try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); @@ -922,21 +922,21 @@ public void testH5Dchunk_iter_all() // offset rather than by position -- the two APIs are not guaranteed to visit chunks in the // same order. java.util.Map byOffset = new java.util.HashMap(); - long sid = -1; + long sid = -1; try { - sid = H5.H5Dget_space(H5did); + sid = H5.H5Dget_space(H5did); long nchunks_by_index = H5.H5Dget_num_chunks(H5did, sid); assertEquals("testH5Dchunk_iter_all: H5Dget_num_chunks agrees", info.getNumChunks(), - nchunks_by_index); + nchunks_by_index); for (int i = 0; i < nchunks_by_index; i++) { - long[] offset = new long[RANK]; - int[] filter_mask = new int[1]; - long[] addr = new long[1]; - long[] size = new long[1]; + long[] offset = new long[RANK]; + int[] filter_mask = new int[1]; + long[] addr = new long[1]; + long[] size = new long[1]; H5.H5Dget_chunk_info(H5did, sid, i, offset, filter_mask, addr, size); byOffset.put(java.util.Arrays.toString(offset), - new long[] {filter_mask[0], addr[0], size[0]}); + new long[] {filter_mask[0], addr[0], size[0]}); } } catch (Throwable err) { @@ -957,11 +957,11 @@ public void testH5Dchunk_iter_all() long[] offset = new long[RANK]; for (int d = 0; d < RANK; d++) offset[d] = info.getOffset(i, d); - String key = java.util.Arrays.toString(offset); + String key = java.util.Arrays.toString(offset); long[] byIndexInfo = byOffset.get(key); assertNotNull("testH5Dchunk_iter_all: no by-index match for offset " + key, byIndexInfo); assertEquals("testH5Dchunk_iter_all: filterMask for offset " + key, byIndexInfo[0], - info.filterMask[i]); + info.filterMask[i]); assertEquals("testH5Dchunk_iter_all: addr for offset " + key, byIndexInfo[1], info.addr[i]); assertEquals("testH5Dchunk_iter_all: size for offset " + key, byIndexInfo[2], info.size[i]); } @@ -979,7 +979,7 @@ public void testH5Dget_chunk_info_by_coord() try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); @@ -1013,7 +1013,7 @@ public void testH5Dget_chunk_storage_size() try { H5.H5Dwrite(H5did, HDF5Constants.H5T_NATIVE_INT, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, write_dset_data); + HDF5Constants.H5P_DEFAULT, write_dset_data); } catch (Exception err) { err.printStackTrace(); @@ -1070,7 +1070,7 @@ public void testH5Dwrite_chunk_and_read_chunk() } byte[] read_buf = new byte[64]; - int filters = -1; + int filters = -1; try { filters = H5.H5Dread_chunk(H5did, HDF5Constants.H5P_DEFAULT, new long[] {0, 0}, read_buf); } diff --git a/java/test/TestH5DChunkIterPerf.java b/java/test/TestH5DChunkIterPerf.java index b91ad7407612..bd2e5e2078db 100644 --- a/java/test/TestH5DChunkIterPerf.java +++ b/java/test/TestH5DChunkIterPerf.java @@ -77,8 +77,8 @@ private long createChunkedDataset(String name, int size) throws Exception H5.H5Pset_chunk(dcpl_id, 2, new long[] {CHUNK, CHUNK}); long sid = H5.H5Screate_simple(2, new long[] {size, size}, null); - long did = H5.H5Dcreate(H5fid, name, HDF5Constants.H5T_NATIVE_UINT8, sid, - HDF5Constants.H5P_DEFAULT, dcpl_id, HDF5Constants.H5P_DEFAULT); + long did = H5.H5Dcreate(H5fid, name, HDF5Constants.H5T_NATIVE_UINT8, sid, HDF5Constants.H5P_DEFAULT, + dcpl_id, HDF5Constants.H5P_DEFAULT); H5.H5Pclose(dcpl_id); H5.H5Sclose(sid); return did; @@ -87,8 +87,7 @@ private long createChunkedDataset(String name, int size) throws Exception private long countByIterate(long did) throws Exception { final long[] count = {0}; - class Data implements H5D_chunk_iter_t { - } + class Data implements H5D_chunk_iter_t {} H5D_chunk_iter_cb cb = new H5D_chunk_iter_cb() { public int apply(MemorySegment offset, int filter_mask, long addr, long size, MemorySegment op_data) @@ -142,9 +141,8 @@ private long countByIndexSharedArena(long did) throws Exception MemorySegment size_segment = arena.allocate(ValueLayout.JAVA_LONG, 1); for (long i = 0; i < nchunks; i++) - org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info(did, sid, i, offset_segment, - filter_mask_segment, addr_segment, - size_segment); + org.hdfgroup.javahdf5.hdf5_h.H5Dget_chunk_info( + did, sid, i, offset_segment, filter_mask_segment, addr_segment, size_segment); } H5.H5Sclose(sid); From cf9500a0ce957f51a251765b1cb67f958d5be5df Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 13:50:13 -0400 Subject: [PATCH 11/15] Add missing JUnit-TestH5DChunkIterPerf.txt reference file (JNI) java/src-jni/test/CMakeLists.txt's HDFTEST_COPY_FILE unconditionally requires testfiles/JUnit-.txt to exist as a build dependency for every entry in HDF5_JAVA_TEST_SOURCES, including TestH5DChunkIterPerf added earlier this session. The file was never committed, so any build with BUILD_TESTING=ON failed outright at the ninja/make generation step with "missing and no known rule to make it" -- this is what was breaking CI broadly (not just the Formatting Check fixed in the previous commits). Committed as empty rather than a captured run's output: this benchmark's printed timings are inherently non-deterministic across machines, and COMPARE_TEST (config/cmake/runExecute.cmake) explicitly skips content comparison when the reference file is empty, checking only exit code -- exactly the right behavior here. Verified locally: clean rebuild no longer fails, and ctest reports "COMPARE Result: 0" / test passed. --- java/src-jni/test/testfiles/JUnit-TestH5DChunkIterPerf.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 java/src-jni/test/testfiles/JUnit-TestH5DChunkIterPerf.txt diff --git a/java/src-jni/test/testfiles/JUnit-TestH5DChunkIterPerf.txt b/java/src-jni/test/testfiles/JUnit-TestH5DChunkIterPerf.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 478414c7b2122b53bd7f49d2215214219308750d Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 13:50:32 -0400 Subject: [PATCH 12/15] Add missing JUnit-TestH5DChunkIterPerf.txt reference file (FFM) Same issue as the JNI-tree commit: java/test/CMakeLists.txt's HDFTEST_COPY_FILE requires testfiles/JUnit-TestH5DChunkIterPerf.txt to exist as a build dependency, and it was never committed, breaking any BUILD_TESTING=ON build. Committed empty for the same reason: this benchmark's output is inherently non-deterministic across machines, and an empty reference file makes COMPARE_TEST check only exit code. --- java/test/testfiles/JUnit-TestH5DChunkIterPerf.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 java/test/testfiles/JUnit-TestH5DChunkIterPerf.txt diff --git a/java/test/testfiles/JUnit-TestH5DChunkIterPerf.txt b/java/test/testfiles/JUnit-TestH5DChunkIterPerf.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 84d5cc633a987bbd69047e5d50ccc02dd775f0b0 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 16:56:30 -0400 Subject: [PATCH 13/15] Fix stale JUnit-TestH5D.txt reference file (JNI) The tracked reference predated this session's new TestH5D test methods (testH5Dget_chunk_info_by_coord, testH5Dget_chunk_storage_size, testH5Dget_chunk_index_type, testH5Dwrite_chunk_and_read_chunk, testH5Dchunk_iter, testH5Dchunk_iter_all, testH5Dread_chunk_buffer_size_mismatch), so JUnit4's hash-based MethodSorters.DEFAULT ordering reshuffled the whole class's dot-progress sequence and the final "OK (32 tests)" count no longer matched, breaking JUnit-TestH5D across every CI job that builds Java tests. Regenerated from an actual local run (44/44 JUnit tests, 178/178 broader ctest passing). Co-Authored-By: Claude Sonnet 5 --- java/src-jni/test/testfiles/JUnit-TestH5D.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java/src-jni/test/testfiles/JUnit-TestH5D.txt b/java/src-jni/test/testfiles/JUnit-TestH5D.txt index 36ce3bd56481..f2b18629eadf 100644 --- a/java/src-jni/test/testfiles/JUnit-TestH5D.txt +++ b/java/src-jni/test/testfiles/JUnit-TestH5D.txt @@ -2,6 +2,7 @@ JUnit version 4.13.2 .testH5DArrayenum_rw .testH5DVLwrVL .testH5Dget_storage_size +.testH5Dget_chunk_info_by_coord .testH5DArraywr .testH5Diterate_write .testH5Dcreate @@ -10,6 +11,7 @@ JUnit version 4.13.2 .testH5DVLwr .testH5Dfill .testH5Dopen +.testH5Dget_chunk_index_type .testH5Dcreate_anon .testH5Dfill_null .testH5Dget_storage_size_empty @@ -17,9 +19,12 @@ JUnit version 4.13.2 .testH5DwriteVL_compound_wrong_member_type .testH5Dget_access_plist .testH5Dread_vlen_of_nested_compound +.testH5Dchunk_iter_all .testH5DArray_string_buffer_flat_StringArray +.testH5Dchunk_iter .testH5Dget_space_closed .testH5DArray_string_buffer_flat_StringArray_write +.testH5Dwrite_chunk_and_read_chunk .testH5DArray_string_buffer .testH5Dread_vlen_of_compound .testH5Dread_compound_of_vlen @@ -28,11 +33,13 @@ JUnit version 4.13.2 .testH5Dvlen_write_read .testH5Dget_space .testH5DwriteVL_vlen_element_not_arraylist +.testH5Dget_chunk_storage_size .testH5Dget_type_closed +.testH5Dread_chunk_buffer_size_mismatch .testH5Dwrite_compound_of_vlen .testH5DwriteVL_undersized_buffer Time: XXXX -OK (32 tests) +OK (39 tests) From 5ba51ff7ead03f74f04343b77220d06d5261d50d Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 21 Jul 2026 18:10:37 -0400 Subject: [PATCH 14/15] Fix stale JUnit-TestH5D.txt reference file (FFM) Same root cause as the JNI-side fix: the tracked reference predated this session's new TestH5D test methods (testH5Dget_chunk_info_by_coord, testH5Dget_chunk_index_type, testH5Dchunk_iter_all, testH5Dchunk_iter, testH5Dwrite_chunk_and_read_chunk, testH5Dget_chunk_storage_size, testH5Dread_chunk_buffer_size_mismatch), so JUnit4's hash-based MethodSorters.DEFAULT ordering reshuffled the whole class's dot-progress sequence and the "OK (32 tests)" count no longer matched. Regenerated from an actual scoped local FFM build/run (JDK 25 + jextract; 44/44 JUnit tests passing), matching the CI failure output exactly. Co-Authored-By: Claude Sonnet 5 --- java/test/testfiles/JUnit-TestH5D.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java/test/testfiles/JUnit-TestH5D.txt b/java/test/testfiles/JUnit-TestH5D.txt index f5c4574e6d5a..cf7378caa6bd 100644 --- a/java/test/testfiles/JUnit-TestH5D.txt +++ b/java/test/testfiles/JUnit-TestH5D.txt @@ -2,6 +2,7 @@ JUnit version 4.13.2 .testH5DArrayenum_rw .testH5DVLwrVL .testH5Dget_storage_size +.testH5Dget_chunk_info_by_coord .testH5DArraywr .testH5Dcreate .testH5Dget_offset @@ -9,16 +10,20 @@ I.testH5Dget_type .testH5DVLwr .testH5Dfill .testH5Dopen +.testH5Dget_chunk_index_type .testH5Dcreate_anon .testH5Dfill_null .testH5Dget_storage_size_empty .testH5DwriteVL_compound_wrong_member_type .testH5Dget_access_plist .testH5Dread_vlen_of_nested_compound +.testH5Dchunk_iter_all .testH5DArray_string_buffer_flat_StringArray .testH5Dvlen_get_buf_size +.testH5Dchunk_iter .testH5Dget_space_closed .testH5DArray_string_buffer_flat_StringArray_write +.testH5Dwrite_chunk_and_read_chunk .testH5DArray_string_buffer .testH5Dread_vlen_of_compound .testH5Dread_compound_of_vlen @@ -27,12 +32,14 @@ I.testH5Dget_type .testH5Dvlen_write_read .testH5Dget_space .testH5DwriteVL_vlen_element_not_arraylist +.testH5Dget_chunk_storage_size .testH5Dget_type_closed +.testH5Dread_chunk_buffer_size_mismatch .testH5Dwrite_compound_of_vlen .testH5Dwrite_readCompound .testH5DwriteVL_undersized_buffer Time: XXXX -OK (32 tests) +OK (39 tests) From 10e5bba2d01aee33a49f1e969e882a9e5997a1a3 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Wed, 22 Jul 2026 17:07:26 -0400 Subject: [PATCH 15/15] Address Copilot review comments on PR #6547 - Fix copy/paste Javadoc ("link callback" -> chunk iterator callback) in both trees' H5D_chunk_iter_t/H5D_chunk_iter_cb interfaces - Use a long loop counter in TestH5D's by-index chunk-info cross-check to avoid truncating a chunk count above Integer.MAX_VALUE - Keep op_data as jobject in H5D_chunk_iter_cb (JNI) instead of casting it to void* before passing it to CallIntMethod --- java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java | 2 +- java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java | 2 +- java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java | 2 +- java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java | 2 +- java/src-jni/jni/h5dImp.c | 2 +- java/src-jni/test/TestH5D.java | 2 +- java/test/TestH5D.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java index 7a9d22885925..dc0f2c028508 100644 --- a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java +++ b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java @@ -19,7 +19,7 @@ import org.hdfgroup.javahdf5.*; /** - * Information class for link callback for H5Dchunk_iter. + * Information class for the chunk iterator callback for H5Dchunk_iter. * */ public interface H5D_chunk_iter_cb extends org.hdfgroup.javahdf5.H5D_chunk_iter_op_t.Function { diff --git a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java index 488640272779..cf27e31e84a2 100644 --- a/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java +++ b/java/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java @@ -13,7 +13,7 @@ package hdf.hdf5lib.callbacks; /** - * Data class for link callback for H5Dchunk_iter. + * Data class for chunk iterator callback for H5Dchunk_iter. * */ public interface H5D_chunk_iter_t { diff --git a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java index 558ab15a9f6d..2e857a90c87d 100644 --- a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java +++ b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_cb.java @@ -13,7 +13,7 @@ package hdf.hdf5lib.callbacks; /** - * Information class for link callback for H5Dchunk_iter. + * Information class for the chunk iterator callback for H5Dchunk_iter. * */ public interface H5D_chunk_iter_cb extends H5Callbacks { diff --git a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java index 488640272779..cf27e31e84a2 100644 --- a/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java +++ b/java/src-jni/hdf/hdf5lib/callbacks/H5D_chunk_iter_t.java @@ -13,7 +13,7 @@ package hdf.hdf5lib.callbacks; /** - * Data class for link callback for H5Dchunk_iter. + * Data class for chunk iterator callback for H5Dchunk_iter. * */ public interface H5D_chunk_iter_t { diff --git a/java/src-jni/jni/h5dImp.c b/java/src-jni/jni/h5dImp.c index cbba28e89e51..516afd18e474 100644 --- a/java/src-jni/jni/h5dImp.c +++ b/java/src-jni/jni/h5dImp.c @@ -2116,7 +2116,7 @@ H5D_chunk_iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsi * invoked from a library-created worker thread). */ JNIEnv *cbenv = wrapper->env; jobject visit_callback = wrapper->visit_callback; - void *op_data = (void *)wrapper->op_data; + jobject op_data = wrapper->op_data; jint status = FAIL; if (NULL == offset) diff --git a/java/src-jni/test/TestH5D.java b/java/src-jni/test/TestH5D.java index 1c8ca13e4d41..d42040805f9f 100644 --- a/java/src-jni/test/TestH5D.java +++ b/java/src-jni/test/TestH5D.java @@ -847,7 +847,7 @@ public void testH5Dchunk_iter_all() assertEquals("testH5Dchunk_iter_all: H5Dget_num_chunks agrees", info.getNumChunks(), nchunks_by_index); - for (int i = 0; i < nchunks_by_index; i++) { + for (long i = 0; i < nchunks_by_index; i++) { long[] offset = new long[RANK]; int[] filter_mask = new int[1]; long[] addr = new long[1]; diff --git a/java/test/TestH5D.java b/java/test/TestH5D.java index 9498de59432e..f419f1511009 100644 --- a/java/test/TestH5D.java +++ b/java/test/TestH5D.java @@ -929,7 +929,7 @@ public void testH5Dchunk_iter_all() assertEquals("testH5Dchunk_iter_all: H5Dget_num_chunks agrees", info.getNumChunks(), nchunks_by_index); - for (int i = 0; i < nchunks_by_index; i++) { + for (long i = 0; i < nchunks_by_index; i++) { long[] offset = new long[RANK]; int[] filter_mask = new int[1]; long[] addr = new long[1];