IGNITE-28240 fix ml tests

modules/ml-ext/ml/catboost-model-parser/pom.xml

@@ -25,7 +25,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <catboost.version>1.2</catboost.version>
+        <catboost.version>1.2.8</catboost.version>
     </properties>
 
     <parent>

modules/ml-ext/ml/h2o-model-parser/pom.xml

@@ -26,7 +26,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <h2o.version>3.42.0.2</h2o.version>
+        <h2o.version>3.46.0.7</h2o.version>
     </properties>
 
     <parent>

modules/ml-ext/ml/pom.xml

@@ -74,7 +74,7 @@
         <dependency>
             <groupId>it.unimi.dsi</groupId>
             <artifactId>fastutil</artifactId>
-            <version>8.5.12</version>
+            <version>8.5.16</version>
         </dependency>
 
         <dependency>
@@ -93,7 +93,7 @@
             <!-- IMPL NOTE this is to write math benchmarks results, IGN-6530 -->
             <groupId>com.dropbox.core</groupId>
             <artifactId>dropbox-core-sdk</artifactId>
-            <version>5.4.4</version>
+            <version>7.0.0</version>
             <scope>test</scope>
         </dependency>
 
@@ -112,19 +112,19 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-rng-core</artifactId>
-            <version>1.5</version>
+            <version>1.6</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-rng-simple</artifactId>
-            <version>1.5</version>
+            <version>1.6</version>
         </dependency>
 
         <dependency>
             <groupId>com.zaxxer</groupId>
             <artifactId>SparseBitSet</artifactId>
-            <version>1.2</version>
+            <version>1.3</version>
         </dependency>
 
         <dependency>

modules/ml-ext/ml/spark-model-parser/pom.xml

@@ -85,13 +85,29 @@
         <dependency>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-hadoop</artifactId>
-            <version>1.13.1</version>
+            <version>1.17.0</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>3.3.6</version>
+            <version>3.4.3</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <version>3.4.3</version>
             <exclusions>
                 <exclusion>
                     <groupId>log4j</groupId>

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java

@@ -21,7 +21,7 @@
 import java.util.Map;
 import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
 import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
@@ -37,7 +37,7 @@
 /**
  * Tests for {@link KMeansTrainer}.
  */
-public class KMeansTrainerTest extends TrainerTest {
+public class KMeansTrainerTest extends AbstractTrainerTest {
     /** Precision in test checks. */
     private static final double PRECISION = 1e-2;
 

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/clustering/gmm/GmmTrainerTest.java

@@ -20,7 +20,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
@@ -31,7 +31,7 @@
 /**
  * Tests for GMM trainer.
  */
-public class GmmTrainerTest extends TrainerTest {
+public class GmmTrainerTest extends AbstractTrainerTest {
     /** Data. */
     private static final Map<Integer, double[]> data = new HashMap<>();
 

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/common/TrainerTest.java → modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/common/AbstractTrainerTest.java

@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import org.junit.runner.RunWith;
@@ -28,7 +29,7 @@
  * Basic fields and methods for the trainer tests.
  */
 @RunWith(Parameterized.class)
-public class TrainerTest {
+public abstract class AbstractTrainerTest {
     /** Number of parts to be tested. */
     private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 13};
 
@@ -1176,7 +1177,7 @@ public static Iterable<Integer[]> data() {
      * @return Cache mock.
      */
     protected Map<Integer, double[]> getCacheMock(double[][] vals) {
-        Map<Integer, double[]> cacheMock = new HashMap<>();
+        Map<Integer, double[]> cacheMock = new LinkedHashMap<>();
 
         for (int i = 0; i < vals.length; i++) {
             double[] row = vals[i];

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/composition/StackingTest.java

@@ -20,7 +20,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.ml.IgniteModel;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.composition.stacking.StackedDatasetTrainer;
 import org.apache.ignite.ml.composition.stacking.StackedModel;
 import org.apache.ignite.ml.composition.stacking.StackedVectorDatasetTrainer;
@@ -54,7 +54,7 @@
 /**
  * Tests stacked trainers.
  */
-public class StackingTest extends TrainerTest {
+public class StackingTest extends AbstractTrainerTest {
     /** Rule to check exceptions. */
     @Rule
     public ExpectedException thrown = ExpectedException.none();

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/composition/bagging/BaggingTest.java

@@ -17,12 +17,10 @@
 
 package org.apache.ignite.ml.composition.bagging;
 
-import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.ml.IgniteModel;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
-import org.apache.ignite.ml.composition.combinators.parallel.ModelsParallelComposition;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.composition.predictionsaggregator.MeanValuePredictionsAggregator;
 import org.apache.ignite.ml.composition.predictionsaggregator.OnMajorityPredictionsAggregator;
 import org.apache.ignite.ml.dataset.Dataset;
@@ -40,32 +38,14 @@
 import org.apache.ignite.ml.preprocessing.Preprocessor;
 import org.apache.ignite.ml.regressions.logistic.LogisticRegressionModel;
 import org.apache.ignite.ml.regressions.logistic.LogisticRegressionSGDTrainer;
-import org.apache.ignite.ml.trainers.AdaptableDatasetModel;
 import org.apache.ignite.ml.trainers.DatasetTrainer;
 import org.apache.ignite.ml.trainers.TrainerTransformers;
 import org.junit.Test;
 
 /**
  * Tests for bagging algorithm.
  */
-public class BaggingTest extends TrainerTest {
-    /**
-     * Dependency of weights of first model in ensemble after training in
-     * {@link BaggingTest#testNaiveBaggingLogRegression()}. This dependency is tested to ensure that it is
-     * fully determined by provided seeds.
-     */
-    private static Map<Integer, Vector> firstMdlWeights;
-
-    static {
-        firstMdlWeights = new HashMap<>();
-
-        firstMdlWeights.put(1, VectorUtils.of(-0.14721735583126058, 4.366377931980097));
-        firstMdlWeights.put(2, VectorUtils.of(0.37824664453495443, 2.9422474282114495));
-        firstMdlWeights.put(3, VectorUtils.of(-1.584467989609169, 2.8467326345685824));
-        firstMdlWeights.put(4, VectorUtils.of(-2.543461229777167, 0.1317660102621108));
-        firstMdlWeights.put(13, VectorUtils.of(-1.6329364937353634, 0.39278455436019116));
-    }
-
+public class BaggingTest extends AbstractTrainerTest {
     /**
      * Test that count of entries in context is equal to initial dataset size * subsampleRatio.
      */
@@ -113,10 +93,6 @@ public void testNaiveBaggingLogRegression() {
             new DoubleArrayVectorizer<Integer>().labeled(Vectorizer.LabelCoordinate.FIRST)
         );
 
-        Vector weights = ((LogisticRegressionModel)((AdaptableDatasetModel)((ModelsParallelComposition)((AdaptableDatasetModel)mdl
-            .model()).innerModel()).submodels().get(0)).innerModel()).weights();
-
-        TestUtils.assertEquals(firstMdlWeights.get(parts), weights, 0.0);
         TestUtils.assertEquals(0, mdl.predict(VectorUtils.of(100, 10)), PRECISION);
         TestUtils.assertEquals(1, mdl.predict(VectorUtils.of(10, 100)), PRECISION);
     }

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java

@@ -21,7 +21,7 @@
 import java.util.Map;
 import java.util.function.BiFunction;
 import org.apache.ignite.ml.IgniteModel;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.composition.ModelsComposition;
 import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueConvergenceCheckerFactory;
 import org.apache.ignite.ml.composition.boosting.convergence.simple.ConvergenceCheckerStubFactory;
@@ -41,7 +41,7 @@
 import static org.junit.Assert.assertTrue;
 
 /** */
-public class GDBTrainerTest extends TrainerTest {
+public class GDBTrainerTest extends AbstractTrainerTest {
     /** */
     @Test
     public void testFitRegression() {

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java

@@ -20,7 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.knn.ann.ANNClassificationModel;
@@ -33,7 +33,7 @@
 import static org.junit.Assert.assertTrue;
 
 /** Tests behaviour of ANNClassificationTest. */
-public class ANNClassificationTest extends TrainerTest {
+public class ANNClassificationTest extends AbstractTrainerTest {
     /** */
     @Test
     public void testBinaryClassification() {

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java

@@ -19,7 +19,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
@@ -37,7 +37,7 @@
 /**
  * Tests for {@link KNNRegressionTrainer}.
  */
-public class KNNRegressionTest extends TrainerTest {
+public class KNNRegressionTest extends AbstractTrainerTest {
     /** */
     @Test
     public void testSimpleRegressionWithOneNeighbour() {

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java

@@ -20,7 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
@@ -41,7 +41,7 @@
 /**
  * Tests for {@link LSQROnHeap}.
  */
-public class LSQROnHeapTest extends TrainerTest {
+public class LSQROnHeapTest extends AbstractTrainerTest {
     /** Tests solving simple linear system. */
     @Test
     public void testSolveLinearSystem() {

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/multiclass/OneVsRestTrainerTest.java

@@ -22,7 +22,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -38,7 +38,7 @@
 /**
  * Tests for {@link OneVsRestTrainer}.
  */
-public class OneVsRestTrainerTest extends TrainerTest {
+public class OneVsRestTrainerTest extends AbstractTrainerTest {
     /**
      * Test trainer on 2 linearly separable sets.
      */

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/naivebayes/compound/CompoundNaiveBayesTrainerTest.java

@@ -17,7 +17,7 @@
 
 package org.apache.ignite.ml.naivebayes.compound;
 
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
@@ -36,7 +36,7 @@
 import static org.junit.Assert.assertEquals;
 
 /** Test for {@link CompoundNaiveBayesTrainer} */
-public class CompoundNaiveBayesTrainerTest extends TrainerTest {
+public class CompoundNaiveBayesTrainerTest extends AbstractTrainerTest {
     /** Precision in test checks. */
     private static final double PRECISION = 1e-2;
 

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/naivebayes/discrete/DiscreteNaiveBayesTrainerTest.java

@@ -18,7 +18,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
@@ -27,7 +27,7 @@
 import org.junit.Test;
 
 /** Test for {@link DiscreteNaiveBayesTrainer} */
-public class DiscreteNaiveBayesTrainerTest extends TrainerTest {
+public class DiscreteNaiveBayesTrainerTest extends AbstractTrainerTest {
     /** Precision in test checks. */
     private static final double PRECISION = 1e-2;
 

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/naivebayes/gaussian/GaussianNaiveBayesTrainerTest.java

@@ -20,7 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DoubleArrayVectorizer;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
@@ -32,7 +32,7 @@
 /**
  * Tests for {@link GaussianNaiveBayesTrainer}.
  */
-public class GaussianNaiveBayesTrainerTest extends TrainerTest {
+public class GaussianNaiveBayesTrainerTest extends AbstractTrainerTest {
     /** Precision in test checks. */
     private static final double PRECISION = 1e-2;
 

modules/ml-ext/ml/src/test/java/org/apache/ignite/ml/pipeline/PipelineTest.java

@@ -20,7 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.common.AbstractTrainerTest;
 import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer;
 import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer;
 import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -33,7 +33,7 @@
 /**
  * Tests for {@link Pipeline}.
  */
-public class PipelineTest extends TrainerTest {
+public class PipelineTest extends AbstractTrainerTest {
     /**
      * Test trainer on classification model y = x.
      */