UNOMI-875: PR3 platform upgrade (Karaf 4.4.11, ES 9.4.3, OS 3.7.0, log4j 2.24)

docker/README.md

@@ -46,25 +46,25 @@ If you want to run it without docker-compose you should then make sure you setup
 For ElasticSearch:
 
 ```bash
-docker pull docker.elastic.co/elasticsearch/elasticsearch:9.2.1
+docker pull docker.elastic.co/elasticsearch/elasticsearch:9.4.3
 docker network create unomi
 docker run -d --name elasticsearch --net unomi -p 9200:9200 -p 9300:9300 \
     -e "discovery.type=single-node" \
     -e "xpack.security.enabled=false" \
     -e cluster.name=contextElasticSearch \ 
-    docker.elastic.co/elasticsearch/elasticsearch:9.2.1
+    docker.elastic.co/elasticsearch/elasticsearch:9.4.3
 ```
 
 For OpenSearch:
 
 ```bash
-docker pull opensearchproject/opensearch:3.0.0
+docker pull opensearchproject/opensearch:3.7.0
 docker network create unomi
 export OPENSEARCH_ADMIN_PASSWORD=enter_your_custom_admin_password_here
 docker run -d --name opensearch --net unomi -p 9200:9200 -p 9300:9300 \
     -e "discovery.type=single-node" \
     -e OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD} \
-    opensearchproject/opensearch:3.0.0
+    opensearchproject/opensearch:3.7.0
 ```
 
 For Unomi (with ElasticSearch):

docker/src/main/docker/docker-compose-build-es.yml

@@ -17,7 +17,7 @@
 version: '2.4'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.3
     volumes:
       - unomi-3-elasticsearch-data:/usr/share/elasticsearch/data
     environment:

docker/src/main/docker/docker-compose-build-os.yml

@@ -23,7 +23,7 @@ networks:
 
 services:
   opensearch-node1:
-    image: opensearchproject/opensearch:3.0.0
+    image: opensearchproject/opensearch:3.7.0
     container_name: opensearch-node1
     environment:
       - cluster.name=opensearch-cluster
@@ -51,7 +51,7 @@ services:
           - opensearch-node1
 
   opensearch-node2:
-    image: opensearchproject/opensearch:3.0.0
+    image: opensearchproject/opensearch:3.7.0
     container_name: opensearch-node2
     environment:
       - cluster.name=opensearch-cluster
@@ -76,7 +76,7 @@ services:
           - opensearch-node2
 
   opensearch-dashboards:
-    image: opensearchproject/opensearch-dashboards:3.0.0
+    image: opensearchproject/opensearch-dashboards:3.7.0
     container_name: opensearch-dashboards
     ports:
       - 5601:5601

docker/src/main/docker/docker-compose-cluster.yml

@@ -17,7 +17,7 @@
 version: '2.4'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.3
     volumes:
       - unomi-3-elasticsearch-data:/usr/share/elasticsearch/data
     environment:

docker/src/main/docker/docker-compose-es.yml

@@ -23,7 +23,7 @@ networks:
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.3
     container_name: elasticsearch
     volumes:
       - unomi-3-elasticsearch-data:/usr/share/elasticsearch/data

docker/src/main/docker/docker-compose-os.yml

@@ -23,7 +23,7 @@ networks:
 
 services:
   opensearch-node1:
-    image: opensearchproject/opensearch:3.0.0
+    image: opensearchproject/opensearch:3.7.0
     container_name: opensearch-node1
     environment:
       - cluster.name=opensearch-cluster
@@ -51,7 +51,7 @@ services:
           - opensearch-node1
 
   opensearch-node2:
-    image: opensearchproject/opensearch:3.0.0
+    image: opensearchproject/opensearch:3.7.0
     container_name: opensearch-node2
     environment:
       - cluster.name=opensearch-cluster
@@ -103,7 +103,7 @@ services:
           - unomi
 
   opensearch-dashboards:
-    image: opensearchproject/opensearch-dashboards:3.0.0
+    image: opensearchproject/opensearch-dashboards:3.7.0
     container_name: opensearch-dashboards
     ports:
       - 5601:5601

itests/docker-compose-snapshot-analysis.yml

@@ -16,7 +16,7 @@
 ################################################################################
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.1.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.4.3
     container_name: es-snapshot-analysis
     environment:
       - discovery.type=single-node

itests/pom.xml

@@ -261,6 +261,17 @@
                     </execution>
                 </executions>
             </plugin>
+            <!-- Pax Exam / pax-url log at DEBUG via pax-logging DefaultServiceLog in the test JVM.
+                 BaseIT already sets org.ops4j.pax.exam / org.ops4j.store to WARN inside Karaf only. -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <systemPropertyVariables>
+                        <org.ops4j.pax.logging.DefaultServiceLog.level>INFO</org.ops4j.pax.logging.DefaultServiceLog.level>
+                    </systemPropertyVariables>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
@@ -293,6 +304,7 @@
                                 <unomi.search.engine>elasticsearch</unomi.search.engine>
                                 <elasticsearch.port>${elasticsearch.port}</elasticsearch.port>
                                 <it.karaf.heap>${karaf.heap}</it.karaf.heap>
+                                <org.ops4j.pax.logging.DefaultServiceLog.level>INFO</org.ops4j.pax.logging.DefaultServiceLog.level>
                             </systemPropertyVariables>
                         </configuration>
                         <executions>
@@ -414,6 +426,7 @@
                                 <my.system.property>foo</my.system.property>
                                 <unomi.search.engine>opensearch</unomi.search.engine>
                                 <org.apache.unomi.opensearch.addresses>localhost:${opensearch.port}</org.apache.unomi.opensearch.addresses>
+                                <org.ops4j.pax.logging.DefaultServiceLog.level>INFO</org.ops4j.pax.logging.DefaultServiceLog.level>
                             </systemPropertyVariables>
                         </configuration>
                         <executions>

itests/src/test/java/org/apache/unomi/itests/BaseIT.java

@@ -17,11 +17,8 @@
 
 package org.apache.unomi.itests;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
 import org.apache.camel.CamelContext;
 import org.apache.camel.Route;
 import org.apache.camel.ServiceStatus;
@@ -143,15 +140,15 @@ public abstract class BaseIT extends KarafTestSupport {
     protected static final String ENABLE_LOG_CHECKING_PROPERTY = "it.unomi.log.checking.enabled";
     protected static final String CAMEL_DEBUG_PROPERTY = "it.unomi.camel.debug";
 
-    protected final static ObjectMapper objectMapper;
     protected static boolean unomiStarted = false;
     protected static String searchEngine = SEARCH_ENGINE_ELASTICSEARCH;
 
-    static {
-        objectMapper = new ObjectMapper();
-        objectMapper.registerModule(new JaxbAnnotationModule());
-        objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
-        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    /**
+     * JSON mapper for IT HTTP/JSON helpers. Initialized on first use (after Unomi features are up),
+     * delegating to the same mapper as the running server.
+     */
+    protected ObjectMapper getObjectMapper() {
+        return CustomObjectMapper.getCustomInstance();
     }
 
     protected PersistenceService persistenceService;
@@ -193,6 +190,9 @@ public abstract class BaseIT extends KarafTestSupport {
     protected LogChecker logChecker;
     private String currentTestName;
 
+
+
+
     public enum AuthType {
         NONE,           // No authentication
         PUBLIC_KEY,     // X-Unomi-Api-Key header with public key
@@ -207,6 +207,20 @@ public enum AuthType {
      * This method should be called early, before any test setup, to ensure
      * the correct search engine is detected and any necessary fixes are applied.
      */
+
+
+    protected TestUtils.RequestResponse executeContextJSONRequest(org.apache.http.client.methods.HttpUriRequest request, String sessionId) throws IOException {
+        return TestUtils.executeContextJSONRequest(request, sessionId, getObjectMapper());
+    }
+
+    protected TestUtils.RequestResponse executeContextJSONRequest(org.apache.http.client.methods.HttpUriRequest request, String sessionId, int expectedStatusCode, boolean withAuth) throws IOException {
+        return TestUtils.executeContextJSONRequest(request, sessionId, expectedStatusCode, withAuth, getObjectMapper());
+    }
+
+    protected TestUtils.RequestResponse executeContextJSONRequest(org.apache.http.client.methods.HttpPost request) throws IOException {
+        return TestUtils.executeContextJSONRequest(request, null, getObjectMapper());
+    }
+
     protected void checkSearchEngine() {
         searchEngine = System.getProperty(SEARCH_ENGINE_PROPERTY, SEARCH_ENGINE_ELASTICSEARCH);
     }
@@ -893,8 +907,8 @@ protected String getValidatedBundleJSON(final String resourcePath, Map<String, S
                 jsonString = jsonString.replace("###" + parameterEntry.getKey() + "###", parameterEntry.getValue());
             }
         }
-        ObjectMapper objectMapper = CustomObjectMapper.getObjectMapper();
-        return objectMapper.writeValueAsString(objectMapper.readTree(jsonString));
+        ObjectMapper mapper = getObjectMapper();
+        return mapper.writeValueAsString(mapper.readTree(jsonString));
     }
 
     /**
@@ -1129,7 +1143,7 @@ protected <T> T get(final String url, Class<T> clazz) {
             final HttpGet httpGet = new HttpGet(getFullUrl(url));
             response = executeHttpRequest(httpGet);
             if (response.getStatusLine().getStatusCode() == 200) {
-                return objectMapper.readValue(response.getEntity().getContent(), clazz);
+                return getObjectMapper().readValue(response.getEntity().getContent(), clazz);
             } else {
                 return null;
             }
@@ -1248,8 +1262,8 @@ protected CloseableHttpResponse executeHttpRequest(HttpUriRequest request, AuthT
     protected String resourceAsString(final String resource) {
         final java.net.URL url = bundleContext.getBundle().getResource(resource);
         try (InputStream stream = url.openStream()) {
-            JsonNode node = objectMapper.readTree(stream);
-            String value = objectMapper.writeValueAsString(node);
+            JsonNode node = getObjectMapper().readTree(stream);
+            String value = getObjectMapper().writeValueAsString(node);
             return value;
         } catch (final Exception e) {
             throw new RuntimeException(e);

itests/src/test/java/org/apache/unomi/itests/BasicIT.java

@@ -32,7 +32,6 @@
 import org.apache.unomi.api.conditions.ConditionType;
 import org.apache.unomi.api.rules.Rule;
 import org.apache.unomi.itests.tools.httpclient.HttpClientThatWaitsForUnomi;
-import org.apache.unomi.persistence.spi.CustomObjectMapper;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -132,7 +131,7 @@ public void testContextJSONWithUrlParameter() throws Exception {
         LOGGER.info("Start test testContextJSONWithUrlParameter");
         ContextRequest contextRequest = new ContextRequest();
         HttpPost request = new HttpPost(getFullUrl("/cxs/context.json?sessionId=" + SESSION_ID_1));
-        request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.create("application/json")));
+        request.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequest), ContentType.create("application/json")));
 
         executeContextJSONRequest(request, SESSION_ID_1);
         LOGGER.info("End test testContextJSONWithUrlParameter");
@@ -144,7 +143,7 @@ public void testContextJSON() throws Exception {
         ContextRequest contextRequest = new ContextRequest();
         contextRequest.setSessionId(SESSION_ID_2);
         HttpPost request = new HttpPost(getFullUrl("/cxs/context.json"));
-        request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.create("application/json")));
+        request.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequest), ContentType.create("application/json")));
 
         executeContextJSONRequest(request, SESSION_ID_2);
         LOGGER.info("End test testContextJSON");
@@ -155,7 +154,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         LOGGER.info("Start test testMultipleLoginOnSameBrowser");
 
         // Add login event condition
-        ConditionType conditionType = CustomObjectMapper.getObjectMapper().readValue(
+        ConditionType conditionType = getObjectMapper().readValue(
                 new File("data/tmp/testLoginEventCondition.json").toURI().toURL(), ConditionType.class);
         definitionsService.setConditionType(conditionType);
 
@@ -171,7 +170,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         );
 
         // Add login rule
-        Rule rule = CustomObjectMapper.getObjectMapper().readValue(new File("data/tmp/testLogin.json").toURI().toURL(),
+        Rule rule = getObjectMapper().readValue(new File("data/tmp/testLogin.json").toURI().toURL(),
                 Rule.class);
         createAndWaitForRule(rule);
 
@@ -181,7 +180,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         // First page view with the first visitor aka VISITOR_1 and SESSION_ID_3
         ContextRequest contextRequestPageViewSession1 = getContextRequestWithPageViewEvent(sourceSite, SESSION_ID_3);
         HttpPost requestPageView1 = new HttpPost(getFullUrl("/cxs/context.json"));
-        requestPageView1.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequestPageViewSession1),
+        requestPageView1.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequestPageViewSession1),
                 ContentType.create("application/json")));
         TestUtils.RequestResponse requestResponsePageView1 = executeContextJSONRequest(requestPageView1, SESSION_ID_3);
         String profileIdVisitor1 = requestResponsePageView1.getContextResponse().getProfileId();
@@ -201,7 +200,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         HttpPost requestLoginVisitor1 = new HttpPost(getFullUrl("/cxs/context.json"));
         requestLoginVisitor1.addHeader("Cookie", requestResponsePageView1.getCookieHeaderValue());
         requestLoginVisitor1.addHeader("X-Unomi-Api-Key", testPublicKey.getKey());
-        requestLoginVisitor1.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequestLoginVisitor1),
+        requestLoginVisitor1.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequestLoginVisitor1),
                 ContentType.create("application/json")));
         TestUtils.RequestResponse requestResponseLoginVisitor1 = executeContextJSONRequest(requestLoginVisitor1, SESSION_ID_3);
         Assert.assertEquals("Context profile id should be the same", profileIdVisitor1,
@@ -214,7 +213,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         // Lets add a page view with VISITOR_1 to simulate reloading the page after login and be able to check the profile properties
         HttpPost requestPageView2 = new HttpPost(getFullUrl("/cxs/context.json"));
         requestPageView2.addHeader("Cookie", requestResponsePageView1.getCookieHeaderValue());
-        requestPageView2.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequestPageViewSession1),
+        requestPageView2.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequestPageViewSession1),
                 ContentType.create("application/json")));
         TestUtils.RequestResponse requestResponsePageView2 = executeContextJSONRequest(requestPageView2, SESSION_ID_3);
         Assert.assertEquals("Context profile id should be the same", profileIdVisitor1,
@@ -229,7 +228,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         ContextRequest contextRequestPageViewSession2 = getContextRequestWithPageViewEvent(sourceSite, SESSION_ID_4);
         HttpPost requestPageView3 = new HttpPost(getFullUrl("/cxs/context.json"));
         requestPageView3.addHeader("Cookie", requestResponsePageView1.getCookieHeaderValue());
-        requestPageView3.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequestPageViewSession2),
+        requestPageView3.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequestPageViewSession2),
                 ContentType.create("application/json")));
         TestUtils.RequestResponse requestResponsePageView3 = executeContextJSONRequest(requestPageView3, SESSION_ID_4);
         Assert.assertEquals("Context profile id should be the same", profileIdVisitor1,
@@ -255,7 +254,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         HttpPost requestLoginVisitor2 = new HttpPost(getFullUrl("/cxs/context.json"));
         requestLoginVisitor2.addHeader("Cookie", requestResponsePageView1.getCookieHeaderValue());
         requestLoginVisitor2.addHeader("X-Unomi-Api-Key", testPublicKey.getKey());
-        requestLoginVisitor2.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequestLoginVisitor2),
+        requestLoginVisitor2.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequestLoginVisitor2),
                 ContentType.create("application/json")));
         TestUtils.RequestResponse requestResponseLoginVisitor2 = executeContextJSONRequest(requestLoginVisitor2, SESSION_ID_4);
         // We should have a new profile id so the session should have been moved from VISITOR_1 to VISITOR_2
@@ -268,7 +267,7 @@ public void testMultipleLoginOnSameBrowser() throws Exception {
         // Lets add a page view with VISITOR_2 to simulate reloading the page after login
         HttpPost requestPageView4 = new HttpPost(getFullUrl("/cxs/context.json"));
         requestPageView4.addHeader("Cookie", requestResponseLoginVisitor2.getCookieHeaderValue());
-        requestPageView4.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequestPageViewSession2),
+        requestPageView4.setEntity(new StringEntity(getObjectMapper().writeValueAsString(contextRequestPageViewSession2),
                 ContentType.create("application/json")));
         TestUtils.RequestResponse requestResponsePageView4 = executeContextJSONRequest(requestPageView4, SESSION_ID_4);
         Assert.assertEquals("Context profile id should be the same", profileIdVisitor2,
@@ -341,9 +340,6 @@ private ContextRequest getContextRequestWithPageViewEvent(CustomItem sourceSite,
         return contextRequest;
     }
 
-    private TestUtils.RequestResponse executeContextJSONRequest(HttpPost request, String sessionId) throws IOException {
-        return TestUtils.executeContextJSONRequest(request, sessionId);
-    }
 
     private void checkVisitor1ResponseProperties(Map<String, Object> profileProperties) {
         checkVisitorResponseProperties(profileProperties, FIRST_NAME_VISITOR_1, LAST_NAME_VISITOR_1, EMAIL_VISITOR_1);