cleanup tests

cli/src/main/kotlin/com/sunya/netchdf/hdf5/BitShuffleFilter.kt → cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/BitShuffleFilter.kt

@@ -1,5 +1,8 @@
-package com.sunya.netchdf.hdf5
+package com.sunya.netchdf.hdf5filters
 
+import com.sunya.netchdf.hdf5.H5filterIF
+import com.sunya.netchdf.hdf5.makeIntFromBEBytes
+import com.sunya.netchdf.hdf5.makeLongFromBEBytes
 import net.jpountz.lz4.LZ4Factory
 
 // seems to handle LZ4_COMPRESSION combined with bit shuffle

cli/src/main/kotlin/com/sunya/netchdf/hdf5/Lz4Filter.kt → cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/Lz4Filter.kt

@@ -1,5 +1,8 @@
-package com.sunya.netchdf.hdf5
+package com.sunya.netchdf.hdf5filters
 
+import com.sunya.netchdf.hdf5.H5filterIF
+import com.sunya.netchdf.hdf5.makeIntFromBEBytes
+import com.sunya.netchdf.hdf5.makeLongFromBEBytes
 import net.jpountz.lz4.LZ4Factory
 import kotlin.math.min
 

cli/src/main/kotlin/com/sunya/netchdf/hdf5/LzfFilter.kt → cli/src/main/kotlin/com/sunya/netchdf/hdf5filters/LzfFilter.kt

@@ -1,7 +1,8 @@
-package com.sunya.netchdf.hdf5
+package com.sunya.netchdf.hdf5filters
 
 import com.ning.compress.lzf.LZFException
 import com.ning.compress.lzf.util.ChunkDecoderFactory
+import com.sunya.netchdf.hdf5.H5filterIF
 
 class LzfFilter() : H5filterIF {
     override fun id() = 32000

cli/src/test/kotlin/com/sunya/netchdf/hdf5/TestFilters.kt

@@ -1,6 +1,9 @@
 package com.sunya.netchdf.hdf5
 
 import com.sunya.cdm.api.computeSize
+import com.sunya.netchdf.hdf5filters.BitShuffleFilter
+import com.sunya.netchdf.hdf5filters.Lz4Filter
+import com.sunya.netchdf.hdf5filters.LzfFilter
 import com.sunya.netchdf.openNetchdfFile
 import kotlin.test.Test
 

core/src/commonMain/kotlin/com/sunya/cdm/api/Variable.kt

@@ -13,7 +13,7 @@ data class Variable<T>(
 ) {
     val name = makeValidCdmObjectName(orgName)
     val rank : Int = dimensions.size
-    val shape : LongArray = dimensions.map { it.length }.toLongArray() // why LongArray ??
+    val shape : LongArray = dimensions.map { it.length }.toLongArray() // if you allow dimensions to be long, you need LongArray shape.
     val nelems : Long = this.shape.computeSize()
 
     fun fullname() : String {

testclibs/build.gradle.kts

@@ -5,8 +5,6 @@ plugins {
 dependencies {
     implementation(project(":core"))
     implementation(project(":testfiles"))
-    // implementation(project(":cli"))
-    //implementation(files("/home/stormy/dev/github/netcdf/netchdf/cli/build/libs/cli.jar"))
 
     implementation(libs.okio)
     implementation(libs.fleeksoft)

testclibs/src/test/kotlin/com/sunya/netchdf/hdf4/H4Ccompare.kt

@@ -7,7 +7,6 @@ import com.sunya.cdm.util.InternalLibraryApi
 import com.sunya.netchdf.*
 import com.sunya.netchdf.hdf4Clib.Hdf4ClibFile
 import com.sunya.netchdf.testfiles.H4Files
-import com.sunya.netchdf.testutils.Stats
 import com.sunya.netchdf.testutils.testData
 import kotlin.test.*
 
@@ -18,13 +17,6 @@ class H4Ccompare {
             return H4Files.files()
         }
 
-        fun afterAll() {
-            if (versions.size > 0) {
-                versions.keys.forEach { println("$it = ${versions[it]!!.size} files") }
-            }
-            Stats.show()
-        }
-
         private val versions = mutableMapOf<String, MutableList<String>>()
     }
 
@@ -135,12 +127,6 @@ class H4Ccompare {
         compareData(testData + "devcdm/hdf4/TOVS_BROWSE_MONTHLY_AM_B861001.E861031_NF.HDF", "Raster_Image_#0")
     }
 
-    @Test
-    fun problemReadData() {
-        val filename = testData + "/devcdm/hdfeos2/MISR_AM1_GP_GMP_P040_O003734_05.eos"
-        readNetchdfData(filename, null, null, true)
-    }
-
     //////////////////////////////////////////////////////////////////////
 
     @Test
@@ -249,11 +235,4 @@ class H4Ccompare {
             compareSelectedDataWithClib(filename) { it.datatype == Datatype.CHAR } //  || it.datatype == Datatype.STRING }
         }
     }
-
-    //@Test
-    fun testIterateWithClib() {
-        files().forEach { filename ->
-            compareIterateWithClib(filename)
-        }
-    }
 }

testclibs/src/test/kotlin/com/sunya/netchdf/hdf5/Hdf5Compare.kt

@@ -15,11 +15,11 @@ import kotlin.test.assertTrue
 class Hdf5Compare {
 
     companion object {
-        @JvmStatic
         fun files(): Iterator<String> {
-            // 10 of 114 fail, because we compare with netcdf4 instead of hdf5 c library
-
-            return H5Files.files()
+            return sequenceOf(
+                N4Files.files().asSequence(),
+                H5Files.files().asSequence(),
+            ).flatten().iterator()
         }
     }
 
@@ -51,6 +51,67 @@ class Hdf5Compare {
         compareDataWithClib(filename)
     }
 
+    // a compound with a member thats a type thats not a seperate typedef.
+    // the obvious thing to do is to be able to add a typedef when processing the member.
+    // or look for it when building H5group
+    @Test
+    fun compoundEnumTypedef() {
+        CompareCdmWithClib(testData + "devcdm/hdf5/enumcmpnd.h5")
+    }
+
+    @Test
+    fun vlenData() {
+        CompareCdmWithClib(testData + "devcdm/netcdf4/tst_vlen_data.nc4")
+        compareDataWithClib(testData + "devcdm/netcdf4/tst_vlen_data.nc4")
+    }
+
+    @Test
+    fun compoundData() {
+        CompareCdmWithClib(testData + "devcdm/netcdf4/tst_compounds.nc4")
+        compareDataWithClib(testData + "devcdm/netcdf4/tst_compounds.nc4")
+    }
+
+    @Test
+    fun stringData() {
+        CompareCdmWithClib(testData + "devcdm/netcdf4/tst_strings.nc")
+        compareDataWithClib(testData + "devcdm/netcdf4/tst_strings.nc")
+    }
+
+    @Test
+    fun opaqueAttribute() {
+        CompareCdmWithClib(testData + "devcdm/netcdf4/tst_opaque_data.nc4")
+    }
+
+    @Test
+    fun testIterateDataSumInfinite() {
+        CompareCdmWithClib(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5")
+        compareDataWithClib(testData + "cdmUnitTest/formats/hdf5/StringsWFilter.h5", varname = "/observation/matrix/data")
+    }
+
+    @Test
+    fun vlstra() {
+        CompareCdmWithClib(testData + "devcdm/hdf5/vlstra.h5")
+    }
+
+    //// the following wont work opening as netcdf
+    @Test
+    fun notNetcdf() {
+        CompareCdmWithClib(testData + "devcdm/hdf5/compound_complex.h5", showCdl = true)
+        CompareCdmWithClib(testData + "devcdm/hdf5/bitfield.h5", showCdl = true)
+        CompareCdmWithClib(testData + "devcdm/hdf5/SDS_array_type.h5", showCdl = true)
+    }
+
+    @Test
+    fun privateTypedef() {
+        CompareCdmWithClib(testData + "devcdm/hdf5/bitop.h5", showCdl = true)
+    }
+
+    @Test
+    fun problemCompareCdl() {
+        CompareCdmWithClib(testData + "devcdm/netcdf4/testNestedStructure.nc", showCdl = true)
+    }
+
+
     ////////////////////////////////////////////////////////////////////
 
     @Test