Use readChunksConcurrent for all btree1 reading.

Readme.md

@@ -1,5 +1,5 @@
 # netchdf
-_last updated: 7/22/2025_
+_last updated: 7/25/2025_
 
 This is a rewrite in Kotlin of parts of the devcdm and netcdf-java libraries. 
 
@@ -133,7 +133,7 @@
 are about 2X slower than native code. Unless the deflate libraries get better, there's not much gain in trying to make
 other parts of the code faster.
 
-We will investigate using Kotlin coroutines to speed up performance bottlenecks.
+We are seeing 10x speedup on data reading. see https://github.com/JohnLCaron/netchdf/issues/189.
 
 
 ### Goals and scope
@@ -262,7 +262,7 @@
     return data as ArrayUByte. 
   * Netcdf-4 encodes CHAR values as HDF5 string type with elemSize = 1, so we use that convention to detect 
     legacy CHAR variables in HDF5 format. (NC_CHAR should not be used in new Netcdf-4 files, use NC_UBYTE or NC_STRING.) 
     Variables of type CHAR return data as STRING, since users can use UBYTE if thats what they intend.
   * Netcdf-4/HDF5 String variables may be fixed or variable length. For fixed Strings, we set the size of Datatype.STRING to 
     the fixed size. For both fixed and variable length Strings, the string will be truncated at the first zero byte, if any.
   * HDF4 does not have a STRING type, but does have signed and unsigned CHAR, and signed and unsigned BYTE. 

core/src/commonMain/kotlin/com/sunya/cdm/api/Netchdf.kt

@@ -20,8 +20,9 @@ interface Netchdf : AutoCloseable {
     // iterate over all the chunks in section, order is arbitrary. TODO where is intersection with wantSection done ??
     fun <T> chunkIterator(v2: Variable<T>, wantSection: SectionPartial? = null, maxElements : Int? = null) : Iterator<ArraySection<T>>
 
+    // iterate over all the chunks in section, order is arbitrary, callbacks are in multiple threads.
     fun <T> readChunksConcurrent(v2: Variable<T>,
-                                 lamda : (ArraySection<*>) -> Unit,
+                                 lamda : (ArraySection<T>) -> Unit,
                                  done : () -> Unit,
                                  wantSection: SectionPartial? = null,
                                  nthreads: Int? = null) {
@@ -30,9 +31,9 @@ interface Netchdf : AutoCloseable {
 }
 
 // the section describes the array chunk reletive to the variable's shape.
-data class ArraySection<T>(val array : ArrayTyped<T>, val section : Section) {
+data class ArraySection<T>(val array : ArrayTyped<T>, val chunkSection : Section) {
     fun intersect(wantSection: SectionPartial) : ArrayTyped<T> {
-        // TODO
+        // TODO ??
         return array
     }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayByte.kt

@@ -37,4 +37,10 @@ class ArrayByte(shape : IntArray, val values: ByteArray) : ArrayTyped<Byte>(Data
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as ByteArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayDouble.kt

@@ -36,4 +36,10 @@ class ArrayDouble(shape : IntArray, val values: DoubleArray) : ArrayTyped<Double
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as DoubleArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayFloat.kt

@@ -37,4 +37,10 @@ class ArrayFloat(shape : IntArray, val values: FloatArray) : ArrayTyped<Float>(D
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as FloatArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayInt.kt

@@ -36,4 +36,10 @@ class ArrayInt(shape : IntArray, val values: IntArray) : ArrayTyped<Int>(Datatyp
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as IntArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayLong.kt

@@ -36,4 +36,10 @@ class ArrayLong(shape : IntArray, val values: LongArray) : ArrayTyped<Long>(Data
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as LongArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayOpaque.kt

@@ -75,5 +75,10 @@ class ArrayOpaque(shape : IntArray, val values : List<ByteArray>, val size : Int
         }
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as MutableList<ByteArray>
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayShort.kt

@@ -36,4 +36,10 @@ class ArrayShort(shape : IntArray, val values: ShortArray) : ArrayTyped<Short>(D
         }
         return dst
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as ShortArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayString.kt

@@ -8,6 +8,7 @@ import com.fleeksoft.charset.decodeToString
 import com.sunya.cdm.api.*
 import com.sunya.cdm.layout.IndexND
 import com.sunya.cdm.layout.IndexSpace
+import com.sunya.cdm.layout.TransferChunk
 
 // fake ByteBuffer
 class ArrayString(shape : IntArray, val values : List<String>) : ArrayTyped<String>(Datatype.STRING, shape) {
@@ -37,6 +38,12 @@ class ArrayString(shape : IntArray, val values : List<String>) : ArrayTyped<Stri
         }
         return ArrayString(section.shape.toIntArray(), sectionList)
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as MutableList<String>
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
 }
 
 /**

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayStructureData.kt

@@ -3,6 +3,7 @@ package com.sunya.cdm.array
 import com.sunya.cdm.api.*
 import com.sunya.cdm.layout.Chunker
 import com.sunya.cdm.layout.IndexSpace
+import com.sunya.cdm.layout.TransferChunk
 
 // fixed length data in the ByteBuffer, var length data goes on the heap
 class ArrayStructureData(shape : IntArray, val ba : ByteArray, val isBE: Boolean, val recsize : Int, val members : List<StructureMember<*>>)
@@ -63,6 +64,10 @@ class ArrayStructureData(shape : IntArray, val ba : ByteArray, val isBE: Boolean
         return ArrayStructureData(section.shape.toIntArray(), sectionBA, isBE, recsize, members)
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+       TODO() // maybe nobody chunks structuredata?
+    }
+
     // structure data is packed into the ByteBuffer starting at the given offset
     // vlens and strings are on the "heap" stored in the parent ArrayStructureData
     inner class StructureData(val ba: ByteArray, val offset: Int, val members: List<StructureMember<*>>) {

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayTyped.kt

@@ -1,6 +1,7 @@
 package com.sunya.cdm.array
 
 import com.sunya.cdm.api.*
+import com.sunya.cdm.layout.TransferChunk
 
 // here, shape must be integers, since size cant exceed 32 bits
 // TODO ArrayTyped<T> is Iterable<T>, but Datatype<T> doesnt have to be T
@@ -46,6 +47,8 @@ abstract class ArrayTyped<T>(val datatype: Datatype<*>, val shape: IntArray) : I
         return dst
     } */
 
+    abstract fun transfer(dst: Any, tc: TransferChunk)
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other !is ArrayTyped<*>) return false
@@ -122,6 +125,10 @@ class ArraySingle<T>(shape : IntArray, datatype : Datatype<*>, fillValueAny : An
     override fun section(section : Section) : ArrayTyped<T> {
         return ArraySingle(section.shape.toIntArray(), datatype, fillValue as Any)
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        // hmmmm could be trouble
+    }
 }
 
 // An empty array of any shape that has no values
@@ -130,6 +137,8 @@ class ArrayEmpty<T>(shape : IntArray, datatype : Datatype<*>) : ArrayTyped<T>(da
     override fun section(section : Section) : ArrayTyped<T> {
         return ArrayEmpty(section.shape.toIntArray(), datatype)
     }
+
+    override fun transfer(dst: Any, tc: TransferChunk) {}
 }
 
 

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayUByte.kt

@@ -41,6 +41,12 @@ class ArrayUByte(shape: IntArray, datatype: Datatype<*>, val values: UByteArray)
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as UByteArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
     companion object {
         fun fromByteArray(shape: IntArray, values: ByteArray): ArrayUByte =
             ArrayUByte(shape, UByteArray(values.size) { values[it].toUByte() })

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayUInt.kt

@@ -40,6 +40,12 @@ class ArrayUInt(shape : IntArray, datatype : Datatype<*>, val values: UIntArray)
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as UIntArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
     companion object {
         fun fromIntArray(shape : IntArray, values : IntArray): ArrayUInt =
             ArrayUInt(shape, UIntArray(values.size) { values[it].toUInt() } )

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayULong.kt

@@ -40,6 +40,12 @@ class ArrayULong(shape : IntArray, datatype : Datatype<*>, val values: ULongArra
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as ULongArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
     companion object {
         fun fromLongArray(shape : IntArray, values : LongArray): ArrayULong =
             ArrayULong(shape, Datatype.ULONG, ULongArray(values.size) { values[it].toULong() } )

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayUShort.kt

@@ -40,6 +40,12 @@ class ArrayUShort(shape : IntArray, datatype : Datatype<*>, val values: UShortAr
         return dst
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as UShortArray
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
     companion object {
         fun fromShortArray(shape : IntArray, values : ShortArray): ArrayUShort =
             ArrayUShort(shape, UShortArray(values.size) { values[it].toUShort() } )

core/src/commonMain/kotlin/com/sunya/cdm/array/ArrayVlen.kt

@@ -3,6 +3,7 @@ package com.sunya.cdm.array
 import com.sunya.cdm.api.*
 import com.sunya.cdm.layout.IndexND
 import com.sunya.cdm.layout.IndexSpace
+import com.sunya.cdm.layout.TransferChunk
 
 // maybe should just return primitive array if only one ??
 class ArrayVlen<T>(shape : IntArray, val values : List<Array<T>>, val baseType : Datatype<T>)
@@ -34,6 +35,12 @@ class ArrayVlen<T>(shape : IntArray, val values : List<Array<T>>, val baseType :
         return ArrayVlen(section.shape.toIntArray(), sectionList, baseType)
     }
 
+    override fun transfer(dst: Any, tc: TransferChunk) {
+        val src = this.values
+        val dest = dst as MutableList<Array<T>>
+        repeat(tc.nelems) { dest[tc.destElem.toInt()+it] = src[tc.srcElem.toInt() + it] }
+    }
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other !is ArrayVlen<*>) return false

core/src/commonMain/kotlin/com/sunya/cdm/layout/Chunker.kt

@@ -102,21 +102,6 @@ internal class Chunker(val dataChunk: IndexSpace, val wantSpace: IndexSpace, mer
         return "Chunker(nelems=$nelems, totalNelems=$totalNelems, dataChunk=$dataChunk, wantSpace=$wantSpace)"
     }
 
-    /* transfer from src to dst buffer, using my computed chunks
-    internal fun transfer(src: ByteBuffer, elemSize : Int, dst: ByteBuffer) {
-        for (chunk in this) {
-            System.arraycopy(
-                src.array(),
-                src.arrayOffset() + elemSize * chunk.srcElem.toInt(),
-                dst.array(),
-                dst.arrayOffset() + elemSize * chunk.destElem.toInt(),
-                elemSize * chunk.nelems,
-            )
-        }
-    }
-
-     */
-
     // the chunker tracks the dst offset
     internal fun transferBA(src: ByteArray, srcOffset: Int, elemSize : Int, dst: ByteArray, dstOffset: Int) {
         for (chunk in this) {

core/src/commonMain/kotlin/com/sunya/cdm/layout/TransferChunk.kt

@@ -5,7 +5,7 @@ package com.sunya.cdm.layout
  * Everything here is in elements, not bytes.
  * Read nelems from src at srcElem, store in destination at destElem.
  */
-internal data class TransferChunk(
+data class TransferChunk(
     val srcElem : Long, // start reading here in the source
     val nelems: Int,    // read these many contiguous elements
     val destElem: Long  // start transferring to here in destination

core/src/commonMain/kotlin/com/sunya/netchdf/hdf4/ODLparser.kt

@@ -7,6 +7,7 @@ import com.sunya.cdm.api.Group
 import com.sunya.cdm.util.Indent
 import com.sunya.cdm.util.InternalLibraryApi
 import com.sunya.cdm.util.makeValidCdmObjectName
+import io.github.oshai.kotlinlogging.KotlinLogging
 
 /*
  * http://newsroom.gsfc.nasa.gov/sdptoolkit/hdfeosfaq.html
@@ -332,10 +333,6 @@ private fun stripQuotes(name: String): String {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
 
-private const val showDetail = false
-private const val showProblems = false
-private const val showValidationFailures = false
-
 @InternalLibraryApi
 class ODLparser(val rootGroup: Group.Builder, val show : Boolean = false) {
 
@@ -398,11 +395,11 @@ class ODLparser(val rootGroup: Group.Builder, val show : Boolean = false) {
                     try {
                         val dimLength = att.component2().toInt()
                         if (dimLength == 0) {
-                            println(" *** ODL has zero dimension length for ${att.component1()}")
+                            logger.warn{" *** ODL has zero dimension length for ${att.component1()}"}
                             return false
                         }
                     } catch (ex : Exception) {
-                        println(" *** ODL cant parse dimension ${att.component1()} length ${att.component2()}")
+                        logger.warn{" *** ODL cant parse dimension ${att.component1()} length ${att.component2()}"}
                         return false
                     }
                 }
@@ -435,4 +432,12 @@ class ODLparser(val rootGroup: Group.Builder, val show : Boolean = false) {
         }
         return true
     }
+
+    companion object {
+        private val logger = KotlinLogging.logger("ODLParser")
+
+        private const val showDetail = false
+        private const val showProblems = false
+        private const val showValidationFailures = false
+    }
 }

core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree1data.kt

@@ -22,6 +22,7 @@ internal class BTree1data(
         rootNode = BTreeNode(rootNodeAddress, null)
     }
 
+    // if other layouts like BTree2data had this interface we could use in chunkConcurrent
     fun asSequence(): Sequence<Pair<Long, DataChunk>> = sequence {
         repeat( tiling.nelems) {
             //val startingIndex = tiling.orderToIndex(it.toLong())
@@ -80,7 +81,7 @@ internal class BTree1data(
         }
 
         // this does not have missing data. Use iterator on the Btree1data class
-        //  return only the leaf nodes, in depth-first order
+        /*  return only the leaf nodes, in depth-first order
         fun asSequence(): Sequence<Pair<Int, DataChunkIF>> = sequence {
             // Handle child nodes recursively (in-order traversal)
             if (children.isNotEmpty()) {
@@ -90,7 +91,7 @@ internal class BTree1data(
             } else {  // If it's a leaf node (no children)
                 keyValues.forEach { yield(it) }
             }
-        }
+        } */
 
         fun findDataChunk(wantOrder: Int): DataChunk? {
             if (children.isNotEmpty()) { // search tree; assumes that chunks are ordered

core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/BTree2data.kt

@@ -326,20 +326,6 @@ internal class BTree2data(private val h5: H5builder, owner: String, address: Lon
         }
     }
 
-    fun makeMissingDataChunkEntry(rootNode: BTree1.Node, wantKey: LongArray): DataChunkIF {
-        return MissingDataChunk()
-    }
-
-    class MissingDataChunk() : DataChunkIF {
-        override fun childAddress() = -1L
-        override fun offsets() = longArrayOf()
-        override fun isMissing() = true
-        override fun chunkSize() = 0
-        override fun filterMask() = 0
-
-        override fun show(tiling : Tiling) : String = "missing"
-    }
-
     companion object {
         internal fun findRecord1byId(records: List<Any>, hugeObjectID: Int): Record1? {
             for (record in records) {