Feature/sync 4.3.0 upstream

.github/workflows/hive-postgres-tpcds-metastore.yml

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build and Publish Postgres TPC-DS Metastore Image
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      imageTag:
+        description: 'Docker image tag. Use "test" for experimentation purposes and proper semantic versioning (e.g., 1.4) when pushing to production. CAUTION: Choose the tag carefully to avoid overwriting existing images.'
+        required: false
+        default: 'test'
+      pushImage:
+        description: 'Push image to Docker Hub? (true/false)'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
+
+      - name: Login to Docker Hub
+        if: ${{ github.event.inputs.pushImage == 'true' }}
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and optionally push Docker image
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294
+        with:
+          context: ./standalone-metastore/metastore-server/docker/hive-postgres-tpcds-metastore/
+          file: ./standalone-metastore/metastore-server/docker/hive-postgres-tpcds-metastore/Dockerfile
+          push: ${{ github.event.inputs.pushImage == 'true' }}
+          tags: ${{ github.repository_owner }}/hive-postgres-tpcds-metastore:${{ github.event.inputs.imageTag }}

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -3868,6 +3868,10 @@ public static enum ConfVars {
         "SSL certificate keystore location for HiveServer2 WebUI."),
     HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PASSWORD("hive.server2.webui.keystore.password", "",
         "SSL certificate keystore password for HiveServer2 WebUI."),
+    HIVE_SERVER2_WEBUI_SSL_KEYSTORE_RELOAD_INTERVAL("hive.server2.webui.keystore.reload.interval", "0",
+        new TimeValidator(TimeUnit.MILLISECONDS),
+        "Interval at which HiveServer2 WebUI checks the SSL keystore file for changes; " +
+        "set to 0 to disable auto-reload. The default is 0."),
     HIVE_SERVER2_WEBUI_SSL_KEYSTORE_TYPE("hive.server2.webui.keystore.type", "",
         "SSL certificate keystore type for HiveServer2 WebUI."),
     HIVE_SERVER2_WEBUI_SSL_INCLUDE_CIPHERSUITES("hive.server2.webui.include.ciphersuites", "",
@@ -5115,6 +5119,11 @@ public static enum ConfVars {
     LLAP_ZK_REGISTRY_NAMESPACE("hive.llap.zk.registry.namespace", null,
         "In the LLAP ZooKeeper-based registry, overrides the ZK path namespace. Note that\n" +
         "using this makes the path management (e.g. setting correct ACLs) your responsibility."),
+    LLAP_CLUSTER_ROUTING_RULES("hive.llap.cluster.routing.rules", "",
+        "Comma-separated rules mapping users/groups to LLAP cluster names.\n" +
+        "Format: user:<name>=<cluster>,group:<name>=<cluster>,default=<cluster>.\n" +
+        "Per-cluster configs are read from hive.llap.cluster.<cluster>.sessions.namespace\n" +
+        "and hive.llap.cluster.<cluster>.registry.namespace."),
     // Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
     // resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
     // These are read by Hadoop IPC, so you should check the usage and naming conventions (e.g.
@@ -5569,7 +5578,8 @@ public static enum ConfVars {
             "hive.iceberg.allow.datafiles.in.table.location.only," +
             "hive.hook.proto.base-directory," +
             "hive.rewrite.data.policy," +
-            "hive.query.history.enabled", // Query History service is initialized on HS2 startup (HIVE-29170)
+            "hive.query.history.enabled," + // Query History service is initialized on HS2 startup (HIVE-29170)
+            "hive.llap.cluster.routing.rules",
         "Comma separated list of configuration options which are immutable at runtime"),
     HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list",
         METASTORE_PWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname

common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java

@@ -445,6 +445,7 @@ public enum ErrorMsg {
   @Deprecated // kept for backwards reference
   REPLACE_VIEW_WITH_MATERIALIZED(10400, "Attempt to replace view {0} with materialized view", true),
   REPLACE_MATERIALIZED_WITH_VIEW(10401, "Attempt to replace materialized view {0} with view", true),
+  VIEW_STORAGE_HANDLER_UNSUPPORTED(10448, "Storage handler {0} doesn't support external logical views", true),
   UPDATE_DELETE_VIEW(10402, "You cannot update or delete records in a view"),
   MATERIALIZED_VIEW_DEF_EMPTY(10403, "Query for the materialized view rebuild could not be retrieved"),
   MERGE_PREDIACTE_REQUIRED(10404, "MERGE statement with both UPDATE and DELETE clauses " +

common/src/java/org/apache/hive/http/HttpServer.java

@@ -35,6 +35,8 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.Timer;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -51,6 +53,7 @@
 import javax.servlet.http.HttpServletRequestWrapper;
 import javax.servlet.http.HttpServletResponse;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 import org.apache.commons.lang3.StringUtils;
@@ -66,6 +69,7 @@
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.security.http.CrossOriginFilter;
+import org.apache.hadoop.security.ssl.FileMonitoringTimerTask;
 import org.apache.hive.http.security.PamAuthenticator;
 import org.apache.hive.http.security.PamConstraint;
 import org.apache.hive.http.security.PamConstraintMapping;
@@ -140,6 +144,8 @@ public class HttpServer {
   private Server webServer;
   private QueuedThreadPool threadPool;
   private PortHandlerWrapper portHandlerWrapper;
+  @VisibleForTesting
+  Timer keystoreChangeMonitor;
 
   /**
    * Create a status server on the given port.
@@ -360,6 +366,10 @@ public void start() throws Exception {
   }
 
   public void stop() throws Exception {
+    if (this.keystoreChangeMonitor != null) {
+      this.keystoreChangeMonitor.cancel();
+      this.keystoreChangeMonitor = null;
+    }
     webServer.stop();
   }
 
@@ -695,6 +705,11 @@ ServerConnector createAndAddChannelConnector(int queueSize, Builder b) {
           new String[excludedSSLProtocols.size()]));
       sslContextFactory.setKeyStorePassword(b.keyStorePassword);
       connector = new ServerConnector(webServer, sslContextFactory, http);
+
+      long reloadInterval = b.conf.getTimeVar(ConfVars.HIVE_SERVER2_WEBUI_SSL_KEYSTORE_RELOAD_INTERVAL, TimeUnit.MILLISECONDS);
+      if (reloadInterval > 0) {
+        this.keystoreChangeMonitor = createKeystoreChangeMonitor(reloadInterval, b.keyStorePath, sslContextFactory);
+      }
     }
 
     connector.setAcceptQueueSize(queueSize);
@@ -706,6 +721,36 @@ ServerConnector createAndAddChannelConnector(int queueSize, Builder b) {
     return connector;
   }
 
+  @VisibleForTesting
+  void setKeystoreChangeMonitor(Timer monitor) {
+    keystoreChangeMonitor = monitor;
+  }
+
+  @VisibleForTesting
+  Timer createKeystoreChangeMonitor(long reloadInterval, String keyStorePath,
+                                    SslContextFactory sslContextFactory) {
+    LOG.info("Starting SSL Certificates Store Monitor. reload interval: {}ms,  keyStorePath: {}", reloadInterval, keyStorePath);
+    Timer timer = new Timer("SSL Certificates Store Monitor", true);
+    //
+    // The Jetty SSLContextFactory provides a 'reload' method which will reload both
+    // truststore and keystore certificates.
+    //
+    timer.schedule(new FileMonitoringTimerTask(
+            Paths.get(keyStorePath),
+            path -> {
+              LOG.info("Reloading certificates from store keystore {}", keyStorePath);
+              try {
+                sslContextFactory.reload(factory -> { });
+              } catch (Exception ex) {
+                LOG.error("Failed to reload SSL keystore certificates", ex);
+              }
+            },null),
+        reloadInterval,
+        reloadInterval
+    );
+    return timer;
+  }
+
   /**
    * Secure the web server with PAM.
    */

common/src/test/org/apache/hive/http/TestHttpServer.java

@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.http;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.util.Timer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.withSettings;
+
+/**
+ * Tests for the SSL keystore auto-reload feature wired in via
+ * {@code HttpServer#makeConfigurationChangeMonitor} and the surrounding
+ * {@code configurationChangeMonitor} field. See HiveConf
+ * {@code hive.server2.webui.keystore.reload.interval}.
+ */
+public class TestHttpServer {
+
+  private Path keystore;
+  private Timer timer;
+
+  @Before
+  public void setUp() throws Exception {
+    keystore = Files.createTempFile("test-keystore-", ".jks");
+    Files.write(keystore, "initial-content".getBytes());
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (timer != null) {
+      timer.cancel();
+    }
+    if (keystore != null) {
+      Files.deleteIfExists(keystore);
+    }
+  }
+
+  /**
+   * When the watched keystore file is modified, the scheduled
+   * {@code FileMonitoringTimerTask} must invoke
+   * {@code SslContextFactory#reload}.
+   */
+  @Test(timeout = 10_000)
+  public void testMonitorReloadsSslContextOnKeystoreModification() throws Exception {
+    SslContextFactory sslContextFactory = mock(SslContextFactory.class);
+    CountDownLatch reloadCalled = new CountDownLatch(1);
+    doAnswer(invocation -> {
+      reloadCalled.countDown();
+      return null;
+    }).when(sslContextFactory).reload(any());
+
+    timer = invokeMakeMonitor(100L, keystore.toString(), sslContextFactory);
+
+    // Bump mtime to guarantee a detected change (FileMonitoringTimerTask compares mtimes).
+    Files.setLastModifiedTime(keystore, FileTime.fromMillis(System.currentTimeMillis() + 5_000));
+
+    assertTrue("SslContextFactory#reload was not called within 5s of keystore mtime change",
+        reloadCalled.await(5, TimeUnit.SECONDS));
+    verify(sslContextFactory, atLeastOnce()).reload(any());
+  }
+
+  /**
+   * Reload failures must be swallowed so a transient bad keystore can't take HS2 down;
+   * the next mtime change should still trigger another reload attempt.
+   */
+  @Test(timeout = 10_000)
+  public void testMonitorSurvivesReloadException() throws Exception {
+    SslContextFactory sslContextFactory = mock(SslContextFactory.class);
+    CountDownLatch reloadCalled = new CountDownLatch(2);
+    doAnswer(invocation -> {
+      reloadCalled.countDown();
+      throw new RuntimeException("simulated keystore reload failure");
+    }).when(sslContextFactory).reload(any());
+
+    timer = invokeMakeMonitor(100L, keystore.toString(), sslContextFactory);
+
+    Files.setLastModifiedTime(keystore, FileTime.fromMillis(System.currentTimeMillis() + 5_000));
+    Thread.sleep(300);
+    Files.setLastModifiedTime(keystore, FileTime.fromMillis(System.currentTimeMillis() + 10_000));
+
+    assertTrue("Monitor should keep firing reload attempts even after exceptions",
+        reloadCalled.await(5, TimeUnit.SECONDS));
+  }
+
+  /**
+   * {@code stop()} must cancel the monitor Timer when one was installed,
+   * so the daemon thread does not outlive HS2.
+   */
+  @Test
+  public void testStopCancelsConfigurationChangeMonitor() throws Exception {
+    HttpServer server = mock(HttpServer.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
+
+    // Track whether cancel() was invoked on the installed timer.
+    boolean[] cancelled = {false};
+    Timer installed = new Timer("test-monitor", true) {
+      @Override
+      public void cancel() {
+        cancelled[0] = true;
+        super.cancel();
+      }
+    };
+    server.setKeystoreChangeMonitor(installed);
+
+    // stop() also calls webServer.stop(); webServer is null on a mock, so we expect
+    // a NullPointerException after the cancel path runs.
+    try {
+      server.stop();
+    } catch (NullPointerException expected) {
+      // intentionally ignored — we only assert the monitor was cancelled
+    }
+    assertTrue("Timer#cancel should have been invoked from stop()", cancelled[0]);
+  }
+
+  /**
+   * No monitor installed → stop() must not blow up trying to cancel a missing Timer.
+   * (Mockito skips field initializers, so we re-establish the production default
+   * {@code Optional.empty()} on the mock before exercising stop().)
+   */
+  @Test
+  public void testStopWithoutMonitorDoesNotThrowFromCancelPath() throws Exception {
+    HttpServer server = mock(HttpServer.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
+    server.setKeystoreChangeMonitor(null);
+    assertNull("keystoreChangeMonitor should be empty for this case", server.keystoreChangeMonitor);
+
+    try {
+      server.stop();
+    } catch (NullPointerException expectedFromWebServerStop) {
+      // ok — the monitor branch must not have thrown before reaching webServer.stop()
+    }
+  }
+
+  // ---- reflection helpers ------------------------------------------------
+
+  private static Timer invokeMakeMonitor(long intervalMs, String keystorePath,
+                                         SslContextFactory sslContextFactory) throws Exception {
+    HttpServer server = mock(HttpServer.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
+    return server.createKeystoreChangeMonitor(intervalMs, keystorePath, sslContextFactory);
+  }
+}