From 539fae508796c767580a11b604f82d17091c490c Mon Sep 17 00:00:00 2001 From: JohnLCaron Date: Tue, 22 Jul 2025 18:29:14 -0600 Subject: [PATCH] seperate chunk iterator tests out --- .../com/sunya/netchdf/testutil/ReadCommon.kt | 161 ----------------- .../kotlin/com/sunya/netchdf/NetchdfTest.kt | 5 +- .../com/sunya/netchdf/hdf4/H4readTest.kt | 5 +- .../netchdf/hdf5/H5readConcurrentTest.kt | 112 ++++++++++++ .../com/sunya/netchdf/hdf5/H5readTest.kt | 107 ----------- .../com/sunya/netchdf/testutils/ReadCommon.kt | 164 +---------------- .../sunya/netchdf/testutils/ReadIterator.kt | 170 ++++++++++++++++++ 7 files changed, 287 insertions(+), 437 deletions(-) create mode 100644 testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt create mode 100644 testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt index 5f42ff74..6a6ea341 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt @@ -124,164 +124,3 @@ fun readMiddleSection(myfile: Netchdf, myvar: Variable<*>, shape: LongArray, sho if (showData) println(mydata) } -////////////////////////////////////////////////////////////////////////////////////// -// compare reading data regular and through the chunkIterate API - -fun compareNetchIterate(filename: String, varname : String? = null, compare : Boolean = true) { - openNetchdfFile(filename).use { myfile -> - if (myfile == null) { - println("*** not a netchdf file = $filename") - return - } - println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") - var countChunks = 0 - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") - countChunks += compareOneVarIterate(myfile, myvar, compare) - } else { - myfile.rootGroup().allVariables().forEach { it -> - countChunks += compareOneVarIterate(myfile, it, compare) - } - } - if (countChunks > 0) { - println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks") - } - } -} - -// compare readArrayData with chunkIterator -fun compareOneVarIterate(myFile: Netchdf, myvar: Variable<*>, compare : Boolean = true) : Int { - val filename = myFile.location().substringAfterLast('/') - val varBytes = myvar.nelems - if (varBytes >= maxBytes) { - println(" *** ${myvar.nameAndShape()} skip reading ArrayData too many bytes= $varBytes max = $maxBytes") - return 0 - } - - val sum1 = AtomicDouble(0.0) - val sumArrayData = if (compare) { - val time3 = measureNanoTime { - val arrayData = myFile.readArrayData(myvar, null) - sumValues(arrayData, sum1) - } - Stats.of("readArrayData", filename, "chunk").accum(time3, 1) - sum1.get() - } else 0.0 - - val sum2 = AtomicDouble(0.0) - var countChunks = 0 - val time1 = measureNanoTime { - val chunkIter = myFile.chunkIterator(myvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array, sum2) - countChunks++ - } - } - val sumChunkIterator = sum2.get() - if (compare) Stats.of("chunkIterator", filename, "chunk").accum(time1, countChunks) - - if (compare && sumChunkIterator.isFinite() && sumArrayData.isFinite()) { - // println(" sumChunkIterator = $sumChunkIterator for ${myvar.nameAndShape()}") - assertTrue(nearlyEquals(sumArrayData, sumChunkIterator), "chunkIterator $sumChunkIterator != $sumArrayData sumArrayData") - } - return countChunks -} - -////////////////////////////////////////////////////////////////////////////////////// -// compare reading data chunkIterate API with Netch and NC - -fun compareIterateWithNC(myfile: Netchdf, ncfile: Netchdf, varname: String?, section: SectionPartial? = null) { - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } - if (myvar == null) { - println(" *** cant find myvar $varname") - return - } - val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() } - if (ncvar == null) { - throw RuntimeException(" *** cant find ncvar $varname") - } - compareOneVarIterate(myvar, myfile, ncvar, ncfile, section) - } else { - myfile.rootGroup().allVariables().forEach { myvar -> - val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() } - if (ncvar == null) { - println(" *** cant find ${myvar.fullname()} in ncfile") - } else { - compareOneVarIterate(myvar, myfile, ncvar, ncfile, null) - } - } - } -} - -fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, ncvar : Variable<*>, ncfile: Netchdf, section: SectionPartial?) { - val sum = AtomicDouble(0.0) - var countChunks = 0 - val time1 = measureNanoTime { - val chunkIter = myfile.chunkIterator(myvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array, sum) - countChunks++ - } - } - Stats.of("netchdf", myfile.location(), "chunk").accum(time1, countChunks) - val sum1 = sum.get() - - sum.set(0.0) - countChunks = 0 - val time2 = measureNanoTime { - val chunkIter = ncfile.chunkIterator(ncvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array, sum) - countChunks++ - } - } - Stats.of("nclib", ncfile.location(), "chunk").accum(time2, countChunks) - val sum2 = sum.get() - - if (sum1.isFinite() && sum2.isFinite()) { - assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") - println("sum = $sum1") - } -} - -/////////////////////////////////////////////////////////// -fun sumValues(array : ArrayTyped<*>, sum : AtomicDouble) { - if (array is ArraySingle || array is ArrayEmpty) { - return // test fillValue the same ?? - } - - if (array.datatype.isNumber) { - for (value in array) { - val number = (value as Number) - val numberd: Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } - } else if (array.datatype.isIntegral) { - for (value in array) { - val useValue = when (value) { - is UByte -> value.toByte() - is UShort -> value.toShort() - is UInt -> value.toInt() - is ULong -> value.toLong() - else -> value - } - val number = (useValue as Number) - val numberd: Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } - } -} - -fun measureNanoTime (block: () -> Unit): Long { - return 0 -} - - diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/NetchdfTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/NetchdfTest.kt index 43ebda7e..ba15bca5 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/NetchdfTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/NetchdfTest.kt @@ -3,7 +3,6 @@ package com.sunya.netchdf import com.sunya.cdm.api.* import com.sunya.netchdf.testfiles.* import com.sunya.netchdf.testutils.Stats -import com.sunya.netchdf.testutils.compareNetchIterate import com.sunya.netchdf.testutils.readNetchdfData import com.sunya.netchdf.testutils.showNetchdfHeader import com.sunya.netchdf.testutils.testData @@ -125,9 +124,9 @@ class NetchdfTest { readNetchdfData(filename) } - // TODO too slow + /* TODO too slow // @Test fun testReadNetchIterate(filename: String) { compareNetchIterate(filename) - } + } */ } \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt index 7655a23d..bd13d6b4 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf4/H4readTest.kt @@ -6,7 +6,6 @@ import com.sunya.netchdf.openNetchdfFile import com.sunya.netchdf.openNetchdfFileWithFormat import com.sunya.netchdf.testfiles.H4Files import com.sunya.netchdf.testutils.Stats -import com.sunya.netchdf.testutils.compareNetchIterate import com.sunya.netchdf.testutils.readNetchdfData import com.sunya.netchdf.testutils.testData import org.junit.jupiter.params.ParameterizedTest @@ -121,10 +120,10 @@ class H4readTest { } } - // takes too long : 47 hours with, 2 hours without (luckily get to divide by 24 for wall clock) + /* takes too long : 47 hours with, 2 hours without (luckily get to divide by 24 for wall clock) // @Test fun testReadIterate(filename: String) { compareNetchIterate(filename) - } + } */ } \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt new file mode 100644 index 00000000..aa721ed6 --- /dev/null +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt @@ -0,0 +1,112 @@ +package com.sunya.netchdf.hdf5 + +import com.sunya.cdm.api.Netchdf +import com.sunya.cdm.api.Variable +import com.sunya.cdm.api.chunkConcurrent +import com.sunya.cdm.array.ArrayTyped +import com.sunya.netchdf.testfiles.H5Files +import com.sunya.netchdf.testutils.AtomicDouble +import com.sunya.netchdf.testutils.Stats +import com.sunya.netchdf.testutils.compareNetchIterate +import com.sunya.netchdf.testutils.measureNanoTime +import kotlin.collections.iterator + +import kotlin.test.* + +// Sanity check read Hdf5File header, for non-netcdf4 files +class H5readConcurrentTest { + + companion object { + @JvmStatic + fun files(): Iterator { + return H5Files.files() + } + } + + @Test + fun testReadIterate() { + files().forEach { filename -> + compareNetchIterate(filename, null) + } + } + + @Test + fun testReadConcurrent() { + files().forEach { filename -> + readH5concurrent(filename, null) + } + } +} + + +fun readH5concurrent(filename: String, varname: String? = null) { + Hdf5File(filename).use { myfile -> + println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") + var countChunks = 0 + if (varname != null) { + val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } + ?: throw RuntimeException("cant find $varname") + countChunks += testOneVarConcurrent(myfile, myvar) + } else { + myfile.rootGroup().allVariables().forEach { it -> + if (it.datatype.isNumber) { + countChunks += testOneVarConcurrent(myfile, it) + } + } + } + if (countChunks > 0) { + println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks") + } + } +} + +fun testOneVarConcurrent(myFile: Netchdf, myvar: Variable<*>): Int { + val filename = myFile.location().substringAfterLast('/') + val sum = AtomicDouble(0.0) + var countChunks = 0 + val time1 = measureNanoTime { + val chunkIter = myFile.chunkIterator(myvar) + for (pair in chunkIter) { + // println(" ${pair.section} = ${pair.array.shape.contentToString()}") + sumValues(pair.array) + countChunks++ + } + } + val sum1 = sum.get() + Stats.of("serialSum", filename, "chunk").accum(time1, countChunks) + + sum.set(0.0) + val time2 = measureNanoTime { + myFile.chunkConcurrent(myvar, null) { sumValues(it.array) } + } + val sum2 = sum.get() + Stats.of("concurrentSum", filename, "chunk").accum(time2, countChunks) + + sum.set(0.0) + val time3 = measureNanoTime { + val arrayData = myFile.readArrayData(myvar, null) + sumValues(arrayData) + } + val sum3 = sum.get() + Stats.of("regularSum", filename, "chunk").accum(time3, countChunks) + + /* if (sum1.isFinite() && sum2.isFinite() && sum3.isFinite()) { + assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") + assertTrue(nearlyEquals(sum1, sum3), "$sum1 != $sum3 sum3") +} + + */ + return countChunks +} + +var sum = AtomicDouble(0.0) +fun sumValues(array: ArrayTyped<*>) { + if (!array.datatype.isNumber or true) return + for (value in array) { + val number = (value as Number) + val numberd: Double = number.toDouble() + if (numberd.isFinite()) { + sum.getAndAdd(numberd) + } + } +} diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt index 7a3a0971..c983e8e5 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readTest.kt @@ -1,19 +1,11 @@ package com.sunya.netchdf.hdf5 -import com.sunya.cdm.api.Netchdf -import com.sunya.cdm.api.Variable -import com.sunya.cdm.api.chunkConcurrent -import com.sunya.cdm.array.ArrayTyped import com.sunya.netchdf.testfiles.H5Files -import com.sunya.netchdf.testutils.AtomicDouble import com.sunya.netchdf.testutils.Stats -import com.sunya.netchdf.testutils.compareNetchIterate -import com.sunya.netchdf.testutils.measureNanoTime import com.sunya.netchdf.testutils.readNetchdfData import com.sunya.netchdf.testutils.testData import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource -import kotlin.collections.iterator import kotlin.test.* @@ -50,20 +42,6 @@ class H5readTest { openH5(testData + "cdmUnitTest/formats/hdf5/groupHasCycle.h5") } - @Test - fun timeIterateConcurrent() { - // readH5(testData + "devcdm/hdf5/zip.h5", "/Data/Compressed_Data") - readH5concurrent(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5", "/observation/matrix/data") - } - - @Test - fun timeIterateProblem() { - compareNetchIterate( - testData + "cdmUnitTest/formats/hdf5/xmdf/mesh_datasets.h5", - "/2DMeshModule/mesh/Datasets/velocity_(64)/Mins" - ) - } - @Test fun testEos() { openH5(testData + "cdmUnitTest/formats/hdf5/aura/MLS-Aura_L2GP-BrO_v01-52-c01_2007d029.he5") @@ -131,20 +109,6 @@ class H5readTest { readNetchdfData(filename) } - //@Test - fun testReadIterate() { - files().forEach { filename -> - compareNetchIterate(filename, null) - } - } - - //@Test - fun testReadConcurrent() { - files().forEach { filename -> - readH5concurrent(filename, null) - } - } - } ///////////////////////////////////////////////////////// @@ -164,75 +128,4 @@ fun openH5(filename: String, varname : String? = null, showCdl : Boolean = false } } return true -} - -fun readH5concurrent(filename: String, varname : String? = null) { - Hdf5File(filename).use { myfile -> - println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") - var countChunks = 0 - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") - countChunks += testOneVarConcurrent(myfile, myvar) - } else { - myfile.rootGroup().allVariables().forEach { it -> - if (it.datatype.isNumber) { - countChunks += testOneVarConcurrent(myfile, it) - } - } - } - if (countChunks > 0) { - println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks") - } - } -} - -fun testOneVarConcurrent(myFile: Netchdf, myvar: Variable<*>) : Int { - val filename = myFile.location().substringAfterLast('/') - sum = AtomicDouble(0.0) - var countChunks = 0 - val time1 = measureNanoTime { - val chunkIter = myFile.chunkIterator(myvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array) - countChunks++ - } - } - val sum1 = sum.get() - Stats.of("serialSum", filename, "chunk").accum(time1, countChunks) - - sum.set(0.0) - val time2 = measureNanoTime { - myFile.chunkConcurrent(myvar, null) { sumValues(it.array) } - } - val sum2 = sum.get() - Stats.of("concurrentSum", filename, "chunk").accum(time2, countChunks) - - sum.set(0.0) - val time3 = measureNanoTime { - val arrayData = myFile.readArrayData(myvar, null) - sumValues(arrayData) - } - val sum3 = sum.get() - Stats.of("regularSum", filename, "chunk").accum(time3, countChunks) - - /* if (sum1.isFinite() && sum2.isFinite() && sum3.isFinite()) { - assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") - assertTrue(nearlyEquals(sum1, sum3), "$sum1 != $sum3 sum3") - } - - */ - return countChunks -} - -var sum = AtomicDouble(0.0) -fun sumValues(array : ArrayTyped<*>) { - if (!array.datatype.isNumber or true) return - for (value in array) { - val number = (value as Number) - val numberd : Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } } \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadCommon.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadCommon.kt index c9a53c79..a1a33200 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadCommon.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadCommon.kt @@ -123,166 +123,4 @@ fun readMiddleSection(myfile: Netchdf, myvar: Variable<*>, shape: LongArray, sho assertTrue(middleShape.equivalent(mydata.shape), "variable ${myvar.name}") } if (showData) println(mydata) -} - -////////////////////////////////////////////////////////////////////////////////////// -// compare reading data regular and through the chunkIterate API - -fun compareNetchIterate(filename: String, varname : String? = null, compare : Boolean = true) { - openNetchdfFile(filename).use { myfile -> - if (myfile == null) { - println("*** not a netchdf file = $filename") - return - } - println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") - var countChunks = 0 - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") - countChunks += compareOneVarIterate(myfile, myvar, compare) - } else { - myfile.rootGroup().allVariables().forEach { it -> - countChunks += compareOneVarIterate(myfile, it, compare) - } - } - if (countChunks > 0) { - println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks") - } - } -} - -// compare readArrayData with chunkIterator -fun compareOneVarIterate(myFile: Netchdf, myvar: Variable<*>, compare : Boolean = true) : Int { - val filename = myFile.location().substringAfterLast('/') - val varBytes = myvar.nelems - if (varBytes >= maxBytes) { - println(" *** ${myvar.nameAndShape()} skip reading ArrayData too many bytes= $varBytes max = $maxBytes") - return 0 - } - - val sum1 = AtomicDouble(0.0) - val sumArrayData = if (compare) { - val time3 = measureNanoTime { - val arrayData = myFile.readArrayData(myvar, null) - sumValues(arrayData, sum1) - } - Stats.of("readArrayData", filename, "chunk").accum(time3, 1) - sum1.get() - } else 0.0 - - val sum2 = AtomicDouble(0.0) - var countChunks = 0 - val time1 = measureNanoTime { - val chunkIter = myFile.chunkIterator(myvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array, sum2) - countChunks++ - } - } - val sumChunkIterator = sum2.get() - if (compare) Stats.of("chunkIterator", filename, "chunk").accum(time1, countChunks) - - if (compare && sumChunkIterator.isFinite() && sumArrayData.isFinite()) { - // println(" sumChunkIterator = $sumChunkIterator for ${myvar.nameAndShape()}") - assertTrue(nearlyEquals(sumArrayData, sumChunkIterator), "chunkIterator $sumChunkIterator != $sumArrayData sumArrayData") - } - return countChunks -} - -////////////////////////////////////////////////////////////////////////////////////// -// compare reading data chunkIterate API with Netch and NC - -fun compareIterateWithNC(myfile: Netchdf, ncfile: Netchdf, varname: String?, section: SectionPartial? = null) { - if (varname != null) { - val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } - if (myvar == null) { - println(" *** cant find myvar $varname") - return - } - val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() } - if (ncvar == null) { - throw RuntimeException(" *** cant find ncvar $varname") - } - compareOneVarIterate(myvar, myfile, ncvar, ncfile, section) - } else { - myfile.rootGroup().allVariables().forEach { myvar -> - val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() } - if (ncvar == null) { - println(" *** cant find ${myvar.fullname()} in ncfile") - } else { - compareOneVarIterate(myvar, myfile, ncvar, ncfile, null) - } - } - } -} - -fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, ncvar : Variable<*>, ncfile: Netchdf, section: SectionPartial?) { - val sum = AtomicDouble(0.0) - var countChunks = 0 - val time1 = measureNanoTime { - val chunkIter = myfile.chunkIterator(myvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array, sum) - countChunks++ - } - } - Stats.of("netchdf", myfile.location(), "chunk").accum(time1, countChunks) - val sum1 = sum.get() - - sum.set(0.0) - countChunks = 0 - val time2 = measureNanoTime { - val chunkIter = ncfile.chunkIterator(ncvar) - for (pair in chunkIter) { - // println(" ${pair.section} = ${pair.array.shape.contentToString()}") - sumValues(pair.array, sum) - countChunks++ - } - } - Stats.of("nclib", ncfile.location(), "chunk").accum(time2, countChunks) - val sum2 = sum.get() - - if (sum1.isFinite() && sum2.isFinite()) { - assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") - println("sum = $sum1") - } -} - -/////////////////////////////////////////////////////////// -fun sumValues(array : ArrayTyped<*>, sum : AtomicDouble) { - if (array is ArraySingle || array is ArrayEmpty) { - return // test fillValue the same ?? - } - - if (array.datatype.isNumber) { - for (value in array) { - val number = (value as Number) - val numberd: Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } - } else if (array.datatype.isIntegral) { - for (value in array) { - val useValue = when (value) { - is UByte -> value.toByte() - is UShort -> value.toShort() - is UInt -> value.toInt() - is ULong -> value.toLong() - else -> value - } - val number = (useValue as Number) - val numberd: Double = number.toDouble() - if (numberd.isFinite()) { - sum.getAndAdd(numberd) - } - } - } -} - -fun measureNanoTime (block: () -> Unit): Long { - return 0 -} - - +} \ No newline at end of file diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt new file mode 100644 index 00000000..8b9c48c9 --- /dev/null +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/testutils/ReadIterator.kt @@ -0,0 +1,170 @@ +package com.sunya.netchdf.testutils + +import com.sunya.cdm.api.* +import com.sunya.cdm.array.* +import com.sunya.cdm.util.nearlyEquals +import com.sunya.netchdf.openNetchdfFile +import kotlin.collections.iterator +import kotlin.test.assertTrue + +////////////////////////////////////////////////////////////////////////////////////// +// compare reading data regular and through the chunkIterate API + +fun compareNetchIterate(filename: String, varname : String? = null, compare : Boolean = true) { + openNetchdfFile(filename).use { myfile -> + if (myfile == null) { + println("*** not a netchdf file = $filename") + return + } + println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") + var countChunks = 0 + if (varname != null) { + val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") + countChunks += compareOneVarIterate(myfile, myvar, compare) + } else { + myfile.rootGroup().allVariables().forEach { it -> + countChunks += compareOneVarIterate(myfile, it, compare) + } + } + if (countChunks > 0) { + println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes chunks = $countChunks") + } + } +} + +// compare readArrayData with chunkIterator +private fun compareOneVarIterate(myFile: Netchdf, myvar: Variable<*>, compare : Boolean = true) : Int { + val filename = myFile.location().substringAfterLast('/') + val varBytes = myvar.nelems + if (varBytes >= maxBytes) { + println(" *** ${myvar.nameAndShape()} skip reading ArrayData too many bytes= $varBytes max = $maxBytes") + return 0 + } + + val sum1 = AtomicDouble(0.0) + val sumArrayData = if (compare) { + val time3 = measureNanoTime { + val arrayData = myFile.readArrayData(myvar, null) + sumValues(arrayData, sum1) + } + Stats.of("readArrayData", filename, "chunk").accum(time3, 1) + sum1.get() + } else 0.0 + + val sum2 = AtomicDouble(0.0) + var countChunks = 0 + val time1 = measureNanoTime { + val chunkIter = myFile.chunkIterator(myvar) + for (pair in chunkIter) { + // println(" ${pair.section} = ${pair.array.shape.contentToString()}") + sumValues(pair.array, sum2) + countChunks++ + } + } + val sumChunkIterator = sum2.get() + if (compare) Stats.of("chunkIterator", filename, "chunk").accum(time1, countChunks) + + if (compare && sumChunkIterator.isFinite() && sumArrayData.isFinite()) { + // println(" sumChunkIterator = $sumChunkIterator for ${myvar.nameAndShape()}") + assertTrue(nearlyEquals(sumArrayData, sumChunkIterator), "chunkIterator $sumChunkIterator != $sumArrayData sumArrayData") + } + return countChunks +} + +////////////////////////////////////////////////////////////////////////////////////// +// compare reading data chunkIterate API with Netch and NC + +fun compareIterateWithNC(myfile: Netchdf, ncfile: Netchdf, varname: String?, section: SectionPartial? = null) { + if (varname != null) { + val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } + if (myvar == null) { + println(" *** cant find myvar $varname") + return + } + val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() } + if (ncvar == null) { + throw RuntimeException(" *** cant find ncvar $varname") + } + compareOneVarIterate(myvar, myfile, ncvar, ncfile, section) + } else { + myfile.rootGroup().allVariables().forEach { myvar -> + val ncvar = ncfile.rootGroup().allVariables().find { it.fullname() == myvar.fullname() } + if (ncvar == null) { + println(" *** cant find ${myvar.fullname()} in ncfile") + } else { + compareOneVarIterate(myvar, myfile, ncvar, ncfile, null) + } + } + } +} + +private fun compareOneVarIterate(myvar: Variable<*>, myfile: Netchdf, ncvar : Variable<*>, ncfile: Netchdf, section: SectionPartial?) { + val sum = AtomicDouble(0.0) + var countChunks = 0 + val time1 = measureNanoTime { + val chunkIter = myfile.chunkIterator(myvar) + for (pair in chunkIter) { + // println(" ${pair.section} = ${pair.array.shape.contentToString()}") + sumValues(pair.array, sum) + countChunks++ + } + } + Stats.of("netchdf", myfile.location(), "chunk").accum(time1, countChunks) + val sum1 = sum.get() + + sum.set(0.0) + countChunks = 0 + val time2 = measureNanoTime { + val chunkIter = ncfile.chunkIterator(ncvar) + for (pair in chunkIter) { + // println(" ${pair.section} = ${pair.array.shape.contentToString()}") + sumValues(pair.array, sum) + countChunks++ + } + } + Stats.of("nclib", ncfile.location(), "chunk").accum(time2, countChunks) + val sum2 = sum.get() + + if (sum1.isFinite() && sum2.isFinite()) { + assertTrue(nearlyEquals(sum1, sum2), "$sum1 != $sum2 sum2") + println("sum = $sum1") + } +} + +/////////////////////////////////////////////////////////// +private fun sumValues(array : ArrayTyped<*>, sum : AtomicDouble) { + if (array is ArraySingle || array is ArrayEmpty) { + return // test fillValue the same ?? + } + + if (array.datatype.isNumber) { + for (value in array) { + val number = (value as Number) + val numberd: Double = number.toDouble() + if (numberd.isFinite()) { + sum.getAndAdd(numberd) + } + } + } else if (array.datatype.isIntegral) { + for (value in array) { + val useValue = when (value) { + is UByte -> value.toByte() + is UShort -> value.toShort() + is UInt -> value.toInt() + is ULong -> value.toLong() + else -> value + } + val number = (useValue as Number) + val numberd: Double = number.toDouble() + if (numberd.isFinite()) { + sum.getAndAdd(numberd) + } + } + } +} + +fun measureNanoTime (block: () -> Unit): Long { + return 0 +} + +