iterator() {
+ return takeRef.get().iterator();
+ }
+
+ // exception that mimics the standard ISE thrown by blocking queues but
+ // embeds a rpc server exception for the client to retry and indicate
+ // if the client should be disconnected.
+ @SuppressWarnings("serial")
+ static class CallQueueOverflowException extends IllegalStateException {
+ private static String TOO_BUSY = "Server too busy";
+ static final CallQueueOverflowException KEEPALIVE =
+ new CallQueueOverflowException(
+ new RetriableException(TOO_BUSY),
+ RpcStatusProto.ERROR);
+ static final CallQueueOverflowException DISCONNECT =
+ new CallQueueOverflowException(
+ new RetriableException(TOO_BUSY + " - disconnecting"),
+ RpcStatusProto.FATAL);
+
+ CallQueueOverflowException(final IOException ioe,
+ final RpcStatusProto status) {
+ super("Queue full", new RpcServerException(ioe.getMessage(), ioe){
+ @Override
+ public RpcStatusProto getRpcStatusProto() {
+ return status;
+ }
+ });
+ }
+ @Override
+ public IOException getCause() {
+ return (IOException)super.getCause();
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/CallerContext.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/CallerContext.java
new file mode 100644
index 000000000000..513142eef9e6
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/CallerContext.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/**
+ * A class defining the caller context for auditing coarse granularity
+ * operations.
+ *
+ * This class is immutable.
+ */
+public final class CallerContext {
+ public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8;
+ /** The caller context.
+ *
+ * It will be truncated if it exceeds the maximum allowed length in
+ * server. The default length limit is
+ * {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT}
+ */
+ private final String context;
+
+ /** The caller's signature for validation.
+ *
+ * The signature is optional. The null or empty signature will be abandoned.
+ * If the signature exceeds the maximum allowed length in server, the caller
+ * context will be abandoned. The default length limit is
+ * {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT}
+ */
+ private final byte[] signature;
+
+ private CallerContext(Builder builder) {
+ this.context = builder.context;
+ this.signature = builder.signature;
+ }
+
+ public String getContext() {
+ return context;
+ }
+
+ public byte[] getSignature() {
+ return signature == null ?
+ null : Arrays.copyOf(signature, signature.length);
+ }
+
+ public boolean isContextValid() {
+ return context != null && !context.isEmpty();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(context).toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ } else if (obj == this) {
+ return true;
+ } else if (obj.getClass() != getClass()) {
+ return false;
+ } else {
+ CallerContext rhs = (CallerContext) obj;
+ return new EqualsBuilder()
+ .append(context, rhs.context)
+ .append(signature, rhs.signature)
+ .isEquals();
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (!isContextValid()) {
+ return "";
+ }
+ String str = context;
+ if (signature != null) {
+ str += ":";
+ str += new String(signature, SIGNATURE_ENCODING);
+ }
+ return str;
+ }
+
+ /** The caller context builder. */
+ public static final class Builder {
+ private final String context;
+ private byte[] signature;
+
+ public Builder(String context) {
+ this.context = context;
+ }
+
+ public Builder setSignature(byte[] signature) {
+ if (signature != null && signature.length > 0) {
+ this.signature = Arrays.copyOf(signature, signature.length);
+ }
+ return this;
+ }
+
+ public CallerContext build() {
+ return new CallerContext(this);
+ }
+ }
+
+ /**
+ * The thread local current caller context.
+ *
+ * Internal class for defered singleton idiom.
+ * https://en.wikipedia.org/wiki/Initialization_on_demand_holder_idiom
+ */
+ private static final class CurrentCallerContextHolder {
+ static final ThreadLocal CALLER_CONTEXT =
+ new InheritableThreadLocal<>();
+ }
+
+ public static CallerContext getCurrent() {
+ return CurrentCallerContextHolder.CALLER_CONTEXT.get();
+ }
+
+ public static void setCurrent(CallerContext callerContext) {
+ CurrentCallerContextHolder.CALLER_CONTEXT.set(callerContext);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/Client.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/Client.java
new file mode 100644
index 000000000000..f1a67df33053
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/Client.java
@@ -0,0 +1,1911 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.security.AccessControlException;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.ipc_.RPC.RpcKind;
+import org.apache.hadoop.ipc_.Server.AuthProtocol;
+import org.apache.hadoop.ipc_.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
+import org.apache.hadoop.net.ConnectTimeoutException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security_.SaslRpcClient;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.AsyncGet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.SocketFactory;
+import javax.security.sasl.Sasl;
+import java.io.*;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.apache.hadoop.ipc_.RpcConstants.CONNECTION_CONTEXT_CALL_ID;
+import static org.apache.hadoop.ipc_.RpcConstants.PING_CALL_ID;
+
+/** A client for an IPC service. IPC calls take a single {@link Writable} as a
+ * parameter, and return a {@link Writable} as their value. A service runs on
+ * a port and is defined by a parameter class and a value class.
+ *
+ * @see Server
+ */
+public class Client implements AutoCloseable {
+ public static final Logger LOG = LoggerFactory.getLogger(Client.class);
+
+ /** A counter for generating call IDs. */
+ private static final AtomicInteger callIdCounter = new AtomicInteger();
+
+ private static final ThreadLocal callId = new ThreadLocal();
+ private static final ThreadLocal retryCount = new ThreadLocal();
+ private static final ThreadLocal EXTERNAL_CALL_HANDLER
+ = new ThreadLocal<>();
+ private static final ThreadLocal>
+ ASYNC_RPC_RESPONSE = new ThreadLocal<>();
+ private static final ThreadLocal asynchronousMode =
+ new ThreadLocal() {
+ @Override
+ protected Boolean initialValue() {
+ return false;
+ }
+ };
+
+ @SuppressWarnings("unchecked")
+ public static AsyncGet
+ getAsyncRpcResponse() {
+ return (AsyncGet) ASYNC_RPC_RESPONSE.get();
+ }
+
+ /**
+ * Set call id and retry count for the next call.
+ * @param cid input cid.
+ * @param rc input rc.
+ * @param externalHandler input externalHandler.
+ */
+ public static void setCallIdAndRetryCount(int cid, int rc,
+ Object externalHandler) {
+ Preconditions.checkArgument(cid != RpcConstants.INVALID_CALL_ID);
+ Preconditions.checkState(callId.get() == null);
+ Preconditions.checkArgument(rc != RpcConstants.INVALID_RETRY_COUNT);
+
+ callId.set(cid);
+ retryCount.set(rc);
+ EXTERNAL_CALL_HANDLER.set(externalHandler);
+ }
+
+ private final ConcurrentMap connections =
+ new ConcurrentHashMap<>();
+ private final Object putLock = new Object();
+ private final Object emptyCondition = new Object();
+ private final AtomicBoolean running = new AtomicBoolean(true);
+
+ private Class valueClass; // class of call values
+ final private Configuration conf;
+
+ private SocketFactory socketFactory; // how to create sockets
+ private final AtomicInteger refCount = new AtomicInteger(1);
+
+ private final int connectionTimeout;
+
+ private final boolean fallbackAllowed;
+ private final boolean bindToWildCardAddress;
+ private final byte[] clientId;
+ private final int maxAsyncCalls;
+ private final AtomicInteger asyncCallCounter = new AtomicInteger(0);
+
+ /**
+ * set the ping interval value in configuration
+ *
+ * @param conf Configuration
+ * @param pingInterval the ping interval
+ */
+ public static final void setPingInterval(Configuration conf,
+ int pingInterval) {
+ conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval);
+ }
+
+ /**
+ * Get the ping interval from configuration;
+ * If not set in the configuration, return the default value.
+ *
+ * @param conf Configuration
+ * @return the ping interval
+ */
+ public static final int getPingInterval(Configuration conf) {
+ return conf.getInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,
+ CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT);
+ }
+
+ /**
+ * The time after which a RPC will timeout.
+ * If ping is not enabled (via ipc.client.ping), then the timeout value is the
+ * same as the pingInterval.
+ * If ping is enabled, then there is no timeout value.
+ *
+ * @param conf Configuration
+ * @return the timeout period in milliseconds. -1 if no timeout value is set
+ * @deprecated use {@link #getRpcTimeout(Configuration)} instead
+ */
+ @Deprecated
+ final public static int getTimeout(Configuration conf) {
+ int timeout = getRpcTimeout(conf);
+ if (timeout > 0) {
+ return timeout;
+ }
+ if (!conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT)) {
+ return getPingInterval(conf);
+ }
+ return -1;
+ }
+
+ /**
+ * The time after which a RPC will timeout.
+ *
+ * @param conf Configuration
+ * @return the timeout period in milliseconds.
+ */
+ public static final int getRpcTimeout(Configuration conf) {
+ int timeout =
+ conf.getInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_DEFAULT);
+ return (timeout < 0) ? 0 : timeout;
+ }
+ /**
+ * set the connection timeout value in configuration
+ *
+ * @param conf Configuration
+ * @param timeout the socket connect timeout value
+ */
+ public static final void setConnectTimeout(Configuration conf, int timeout) {
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY, timeout);
+ }
+
+ /**
+ * Increment this client's reference count
+ */
+ void incCount() {
+ refCount.incrementAndGet();
+ }
+
+ /**
+ * Decrement this client's reference count
+ */
+ int decAndGetCount() {
+ return refCount.decrementAndGet();
+ }
+
+ /** Check the rpc response header. */
+ void checkResponse(RpcResponseHeaderProto header) throws IOException {
+ if (header == null) {
+ throw new EOFException("Response is null.");
+ }
+ if (header.hasClientId()) {
+ // check client IDs
+ final byte[] id = header.getClientId().toByteArray();
+ if (!Arrays.equals(id, RpcConstants.DUMMY_CLIENT_ID)) {
+ if (!Arrays.equals(id, clientId)) {
+ throw new IOException("Client IDs not matched: local ID="
+ + StringUtils.byteToHexString(clientId) + ", ID in response="
+ + StringUtils.byteToHexString(header.getClientId().toByteArray()));
+ }
+ }
+ }
+ }
+
+ Call createCall(RPC.RpcKind rpcKind, Writable rpcRequest) {
+ return new Call(rpcKind, rpcRequest);
+ }
+
+ /**
+ * Class that represents an RPC call
+ */
+ static class Call {
+ final int id; // call id
+ final int retry; // retry count
+ final Writable rpcRequest; // the serialized rpc request
+ Writable rpcResponse; // null if rpc has error
+ IOException error; // exception, null if success
+ final RPC.RpcKind rpcKind; // Rpc EngineKind
+ boolean done; // true when call is done
+ private final Object externalHandler;
+ private AlignmentContext alignmentContext;
+
+ private Call(RPC.RpcKind rpcKind, Writable param) {
+ this.rpcKind = rpcKind;
+ this.rpcRequest = param;
+
+ final Integer id = callId.get();
+ if (id == null) {
+ this.id = nextCallId();
+ } else {
+ callId.set(null);
+ this.id = id;
+ }
+
+ final Integer rc = retryCount.get();
+ if (rc == null) {
+ this.retry = 0;
+ } else {
+ this.retry = rc;
+ }
+
+ this.externalHandler = EXTERNAL_CALL_HANDLER.get();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + id;
+ }
+
+ /** Indicate when the call is complete and the
+ * value or error are available. Notifies by default. */
+ protected synchronized void callComplete() {
+ this.done = true;
+ notify(); // notify caller
+
+ if (externalHandler != null) {
+ synchronized (externalHandler) {
+ externalHandler.notify();
+ }
+ }
+ }
+
+ /**
+ * Set an AlignmentContext for the call to update when call is done.
+ *
+ * @param ac alignment context to update.
+ */
+ public synchronized void setAlignmentContext(AlignmentContext ac) {
+ this.alignmentContext = ac;
+ }
+
+ /** Set the exception when there is an error.
+ * Notify the caller the call is done.
+ *
+ * @param error exception thrown by the call; either local or remote
+ */
+ public synchronized void setException(IOException error) {
+ this.error = error;
+ callComplete();
+ }
+
+ /** Set the return value when there is no error.
+ * Notify the caller the call is done.
+ *
+ * @param rpcResponse return value of the rpc call.
+ */
+ public synchronized void setRpcResponse(Writable rpcResponse) {
+ this.rpcResponse = rpcResponse;
+ callComplete();
+ }
+
+ public synchronized Writable getRpcResponse() {
+ return rpcResponse;
+ }
+ }
+
+ /** Thread that reads responses and notifies callers. Each connection owns a
+ * socket connected to a remote address. Calls are multiplexed through this
+ * socket: responses may be delivered out of order. */
+ private class Connection extends Thread {
+ private InetSocketAddress server; // server ip:port
+ private final ConnectionId remoteId; // connection id
+ private AuthMethod authMethod; // authentication method
+ private AuthProtocol authProtocol;
+ private int serviceClass;
+ private SaslRpcClient saslRpcClient;
+
+ private Socket socket = null; // connected socket
+ private IpcStreams ipcStreams;
+ private final int maxResponseLength;
+ private final int rpcTimeout;
+ private int maxIdleTime; //connections will be culled if it was idle for
+ //maxIdleTime msecs
+ private final RetryPolicy connectionRetryPolicy;
+ private final int maxRetriesOnSasl;
+ private int maxRetriesOnSocketTimeouts;
+ private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+ private final boolean tcpLowLatency; // if T then use low-delay QoS
+ private final boolean doPing; //do we need to send ping message
+ private final int pingInterval; // how often sends ping to the server
+ private final int soTimeout; // used by ipc ping and rpc timeout
+ private byte[] pingRequest; // ping message
+
+ // currently active calls
+ private Hashtable calls = new Hashtable();
+ private AtomicLong lastActivity = new AtomicLong();// last I/O activity time
+ private AtomicBoolean shouldCloseConnection = new AtomicBoolean(); // indicate if the connection is closed
+ private IOException closeException; // close reason
+
+ private final Thread rpcRequestThread;
+ private final SynchronousQueue> rpcRequestQueue =
+ new SynchronousQueue<>(true);
+
+ private AtomicReference connectingThread = new AtomicReference<>();
+ private final Consumer removeMethod;
+
+ Connection(ConnectionId remoteId, int serviceClass,
+ Consumer removeMethod) {
+ this.remoteId = remoteId;
+ this.server = remoteId.getAddress();
+ this.rpcRequestThread = new Thread(new RpcRequestSender(),
+ "IPC Parameter Sending Thread for " + remoteId);
+ this.rpcRequestThread.setDaemon(true);
+
+ this.maxResponseLength = remoteId.conf.getInt(
+ CommonConfigurationKeys.IPC_MAXIMUM_RESPONSE_LENGTH,
+ CommonConfigurationKeys.IPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT);
+ this.rpcTimeout = remoteId.getRpcTimeout();
+ this.maxIdleTime = remoteId.getMaxIdleTime();
+ this.connectionRetryPolicy = remoteId.connectionRetryPolicy;
+ this.maxRetriesOnSasl = remoteId.getMaxRetriesOnSasl();
+ this.maxRetriesOnSocketTimeouts = remoteId.getMaxRetriesOnSocketTimeouts();
+ this.tcpNoDelay = remoteId.getTcpNoDelay();
+ this.tcpLowLatency = remoteId.getTcpLowLatency();
+ this.doPing = remoteId.getDoPing();
+ if (doPing) {
+ // construct a RPC header with the callId as the ping callId
+ ResponseBuffer buf = new ResponseBuffer();
+ RpcRequestHeaderProto pingHeader = ProtoUtil
+ .makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
+ OperationProto.RPC_FINAL_PACKET, PING_CALL_ID,
+ RpcConstants.INVALID_RETRY_COUNT, clientId);
+ try {
+ pingHeader.writeDelimitedTo(buf);
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to write to buf for "
+ + remoteId + " in " + Client.this + " due to " + e, e);
+ }
+ pingRequest = buf.toByteArray();
+ }
+ this.pingInterval = remoteId.getPingInterval();
+ if (rpcTimeout > 0) {
+ // effective rpc timeout is rounded up to multiple of pingInterval
+ // if pingInterval < rpcTimeout.
+ this.soTimeout = (doPing && pingInterval < rpcTimeout) ?
+ pingInterval : rpcTimeout;
+ } else {
+ this.soTimeout = pingInterval;
+ }
+ this.serviceClass = serviceClass;
+ this.removeMethod = removeMethod;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The ping interval is " + this.pingInterval + " ms.");
+ }
+
+ UserGroupInformation ticket = remoteId.getTicket();
+ // try SASL if security is enabled or if the ugi contains tokens.
+ // this causes a SIMPLE client with tokens to attempt SASL
+ boolean trySasl = UserGroupInformation.isSecurityEnabled() ||
+ (ticket != null && !ticket.getTokens().isEmpty());
+ this.authProtocol = trySasl ? AuthProtocol.SASL : AuthProtocol.NONE;
+
+ this.setName("IPC Client (" + socketFactory.hashCode() +") connection to " +
+ server.toString() +
+ " from " + ((ticket==null)?"an unknown user":ticket.getUserName()));
+ this.setDaemon(true);
+ }
+
+ /** Update lastActivity with the current time. */
+ private void touch() {
+ lastActivity.set(Time.now());
+ }
+
+ /**
+ * Add a call to this connection's call queue and notify
+ * a listener; synchronized.
+ * Returns false if called during shutdown.
+ * @param call to add
+ * @return true if the call was added.
+ */
+ private synchronized boolean addCall(Call call) {
+ if (shouldCloseConnection.get())
+ return false;
+ calls.put(call.id, call);
+ notify();
+ return true;
+ }
+
+ /** This class sends a ping to the remote side when timeout on
+ * reading. If no failure is detected, it retries until at least
+ * a byte is read.
+ */
+ private class PingInputStream extends FilterInputStream {
+ /* constructor */
+ protected PingInputStream(InputStream in) {
+ super(in);
+ }
+
+ /* Process timeout exception
+ * if the connection is not going to be closed or
+ * the RPC is not timed out yet, send a ping.
+ */
+ private void handleTimeout(SocketTimeoutException e, int waiting)
+ throws IOException {
+ if (shouldCloseConnection.get() || !running.get() ||
+ (0 < rpcTimeout && rpcTimeout <= waiting)) {
+ throw e;
+ } else {
+ sendPing();
+ }
+ }
+
+ /** Read a byte from the stream.
+ * Send a ping if timeout on read. Retries if no failure is detected
+ * until a byte is read.
+ * @throws IOException for any IO problem other than socket timeout
+ */
+ @Override
+ public int read() throws IOException {
+ int waiting = 0;
+ do {
+ try {
+ return super.read();
+ } catch (SocketTimeoutException e) {
+ waiting += soTimeout;
+ handleTimeout(e, waiting);
+ }
+ } while (true);
+ }
+
+ /** Read bytes into a buffer starting from offset off
+ * Send a ping if timeout on read. Retries if no failure is detected
+ * until a byte is read.
+ *
+ * @return the total number of bytes read; -1 if the connection is closed.
+ */
+ @Override
+ public int read(byte[] buf, int off, int len) throws IOException {
+ int waiting = 0;
+ do {
+ try {
+ return super.read(buf, off, len);
+ } catch (SocketTimeoutException e) {
+ waiting += soTimeout;
+ handleTimeout(e, waiting);
+ }
+ } while (true);
+ }
+ }
+
+ private synchronized void disposeSasl() {
+ if (saslRpcClient != null) {
+ try {
+ saslRpcClient.dispose();
+ saslRpcClient = null;
+ } catch (IOException ignored) {
+ }
+ }
+ }
+
+ private synchronized boolean shouldAuthenticateOverKrb() throws IOException {
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ UserGroupInformation realUser = currentUser.getRealUser();
+ if (authMethod == AuthMethod.KERBEROS && loginUser != null &&
+ // Make sure user logged in using Kerberos either keytab or TGT
+ loginUser.hasKerberosCredentials() &&
+ // relogin only in case it is the login user (e.g. JT)
+ // or superuser (like oozie).
+ (loginUser.equals(currentUser) || loginUser.equals(realUser))) {
+ return true;
+ }
+ return false;
+ }
+
+ private synchronized AuthMethod setupSaslConnection(IpcStreams streams)
+ throws IOException {
+ // Do not use Client.conf here! We must use ConnectionId.conf, since the
+ // Client object is cached and shared between all RPC clients, even those
+ // for separate services.
+ saslRpcClient = new SaslRpcClient(remoteId.getTicket(),
+ remoteId.getProtocol(), remoteId.getAddress(), remoteId.conf);
+ return saslRpcClient.saslConnect(streams);
+ }
+
+ /**
+ * Update the server address if the address corresponding to the host
+ * name has changed.
+ *
+ * @return true if an addr change was detected.
+ * @throws IOException when the hostname cannot be resolved.
+ */
+ private synchronized boolean updateAddress() throws IOException {
+ // Do a fresh lookup with the old host name.
+ InetSocketAddress currentAddr = NetUtils.createSocketAddrForHost(
+ server.getHostName(), server.getPort());
+
+ if (!currentAddr.isUnresolved() && !server.equals(currentAddr)) {
+ LOG.warn("Address change detected. Old: {} New: {}", server, currentAddr);
+ server = currentAddr;
+ // Update the remote address so that reconnections are with the updated address.
+ // This avoids thrashing.
+ remoteId.setAddress(currentAddr);
+ UserGroupInformation ticket = remoteId.getTicket();
+ this.setName("IPC Client (" + socketFactory.hashCode()
+ + ") connection to " + server.toString() + " from "
+ + ((ticket == null) ? "an unknown user" : ticket.getUserName()));
+ return true;
+ }
+ return false;
+ }
+
+ private synchronized void setupConnection(
+ UserGroupInformation ticket) throws IOException {
+ LOG.debug("Setup connection to " + server.toString());
+ short ioFailures = 0;
+ short timeoutFailures = 0;
+ while (true) {
+ try {
+ if (server.isUnresolved()) {
+ // Jump into the catch block. updateAddress() will re-resolve
+ // the address if this is just a temporary DNS failure. If not,
+ // it will timeout after max ipc client retries
+ throw NetUtils.wrapException(server.getHostName(),
+ server.getPort(),
+ NetUtils.getHostname(),
+ 0,
+ new UnknownHostException());
+ }
+ this.socket = socketFactory.createSocket();
+ this.socket.setTcpNoDelay(tcpNoDelay);
+ this.socket.setKeepAlive(true);
+
+ if (tcpLowLatency) {
+ /*
+ * This allows intermediate switches to shape IPC traffic
+ * differently from Shuffle/HDFS DataStreamer traffic.
+ *
+ * IPTOS_RELIABILITY (0x04) | IPTOS_LOWDELAY (0x10)
+ *
+ * Prefer to optimize connect() speed & response latency over net
+ * throughput.
+ */
+ this.socket.setTrafficClass(0x04 | 0x10);
+ this.socket.setPerformancePreferences(1, 2, 0);
+ }
+
+ /*
+ * Bind the socket to the host specified in the principal name of the
+ * client, to ensure Server matching address of the client connection
+ * to host name in principal passed.
+ */
+ InetSocketAddress bindAddr = null;
+ if (ticket != null && ticket.hasKerberosCredentials()) {
+ KerberosInfo krbInfo =
+ remoteId.getProtocol().getAnnotation(KerberosInfo.class);
+ if (krbInfo != null) {
+ String principal = ticket.getUserName();
+ String host = SecurityUtil.getHostFromPrincipal(principal);
+ // If host name is a valid local address then bind socket to it
+ InetAddress localAddr = NetUtils.getLocalInetAddress(host);
+ if (localAddr != null) {
+ this.socket.setReuseAddress(true);
+ InetAddress bindTo = bindToWildCardAddress ? null : localAddr;
+ LOG.debug("Binding {} to {}", principal,
+ (bindToWildCardAddress) ? "0.0.0.0" : localAddr);
+ this.socket.bind(new InetSocketAddress(localAddr, 0));
+ }
+ }
+ }
+
+ NetUtils.connect(this.socket, server, bindAddr, connectionTimeout);
+ this.socket.setSoTimeout(soTimeout);
+ return;
+ } catch (ConnectTimeoutException toe) {
+ /* Check for an address change and update the local reference.
+ * Reset the failure counter if the address was changed
+ */
+ if (updateAddress()) {
+ timeoutFailures = ioFailures = 0;
+ }
+ handleConnectionTimeout(timeoutFailures++,
+ maxRetriesOnSocketTimeouts, toe);
+ } catch (IOException ie) {
+ if (updateAddress()) {
+ timeoutFailures = ioFailures = 0;
+ try {
+ // HADOOP-17068: when server changed, ignore the exception.
+ handleConnectionFailure(ioFailures++, ie);
+ } catch (IOException ioe) {
+ LOG.warn("Exception when handle ConnectionFailure: "
+ + ioe.getMessage());
+ }
+ } else {
+ handleConnectionFailure(ioFailures++, ie);
+ }
+ }
+ }
+ }
+
+ /**
+ * If multiple clients with the same principal try to connect to the same
+ * server at the same time, the server assumes a replay attack is in
+ * progress. This is a feature of kerberos. In order to work around this,
+ * what is done is that the client backs off randomly and tries to initiate
+ * the connection again. The other problem is to do with ticket expiry. To
+ * handle that, a relogin is attempted.
+ */
+ private synchronized void handleSaslConnectionFailure(
+ final int currRetries, final int maxRetries, final IOException ex,
+ final Random rand, final UserGroupInformation ugi) throws IOException,
+ InterruptedException {
+ ugi.doAs(new PrivilegedExceptionAction() {
+ @Override
+ public Object run() throws IOException, InterruptedException {
+ final short MAX_BACKOFF = 5000;
+ closeConnection();
+ disposeSasl();
+ if (shouldAuthenticateOverKrb()) {
+ if (currRetries < maxRetries) {
+ LOG.debug("Exception encountered while connecting to the server {}", remoteId, ex);
+ // try re-login
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else if (UserGroupInformation.isLoginTicketBased()) {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ // have granularity of milliseconds
+ //we are sleeping with the Connection lock held but since this
+ //connection instance is being used for connecting to the server
+ //in question, it is okay
+ Thread.sleep((rand.nextInt(MAX_BACKOFF) + 1));
+ return null;
+ } else {
+ String msg = "Couldn't setup connection for "
+ + UserGroupInformation.getLoginUser().getUserName() + " to "
+ + remoteId;
+ LOG.warn(msg, ex);
+ throw NetUtils.wrapException(remoteId.getAddress().getHostName(),
+ remoteId.getAddress().getPort(),
+ NetUtils.getHostname(),
+ 0,
+ ex);
+ }
+ } else {
+ // With RequestHedgingProxyProvider, one rpc call will send multiple
+ // requests to all namenodes. After one request return successfully,
+ // all other requests will be interrupted. It's not a big problem,
+ // and should not print a warning log.
+ if (ex instanceof InterruptedIOException) {
+ LOG.debug("Exception encountered while connecting to the server {}", remoteId, ex);
+ } else {
+ LOG.warn("Exception encountered while connecting to the server {}", remoteId, ex);
+ }
+ }
+ if (ex instanceof RemoteException)
+ throw (RemoteException) ex;
+ throw new IOException(ex);
+ }
+ });
+ }
+
+
+ /** Connect to the server and set up the I/O streams. It then sends
+ * a header to the server and starts
+ * the connection thread that waits for responses.
+ */
+ private synchronized void setupIOstreams(
+ AtomicBoolean fallbackToSimpleAuth) {
+ try {
+ if (socket != null || shouldCloseConnection.get()) {
+ setFallBackToSimpleAuth(fallbackToSimpleAuth);
+ return;
+ }
+ UserGroupInformation ticket = remoteId.getTicket();
+ if (ticket != null) {
+ final UserGroupInformation realUser = ticket.getRealUser();
+ if (realUser != null) {
+ ticket = realUser;
+ }
+ }
+ connectingThread.set(Thread.currentThread());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to "+server);
+ }
+ short numRetries = 0;
+ Random rand = null;
+ while (true) {
+ setupConnection(ticket);
+ ipcStreams = new IpcStreams(socket, maxResponseLength);
+ writeConnectionHeader(ipcStreams);
+ if (authProtocol == AuthProtocol.SASL) {
+ try {
+ authMethod = ticket
+ .doAs(new PrivilegedExceptionAction() {
+ @Override
+ public AuthMethod run()
+ throws IOException, InterruptedException {
+ return setupSaslConnection(ipcStreams);
+ }
+ });
+ } catch (IOException ex) {
+ if (saslRpcClient == null) {
+ // whatever happened -it can't be handled, so rethrow
+ throw ex;
+ }
+ // otherwise, assume a connection problem
+ authMethod = saslRpcClient.getAuthMethod();
+ if (rand == null) {
+ rand = new Random();
+ }
+ handleSaslConnectionFailure(numRetries++, maxRetriesOnSasl, ex,
+ rand, ticket);
+ continue;
+ }
+ if (authMethod != AuthMethod.SIMPLE) {
+ // Sasl connect is successful. Let's set up Sasl i/o streams.
+ ipcStreams.setSaslClient(saslRpcClient);
+ // for testing
+ remoteId.saslQop =
+ (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP);
+ LOG.debug("Negotiated QOP is :" + remoteId.saslQop);
+ }
+ setFallBackToSimpleAuth(fallbackToSimpleAuth);
+ }
+
+ if (doPing) {
+ ipcStreams.setInputStream(new PingInputStream(ipcStreams.in));
+ }
+
+ writeConnectionContext(remoteId, authMethod);
+
+ // update last activity time
+ touch();
+
+ // start the receiver thread after the socket connection has been set
+ // up
+ start();
+ return;
+ }
+ } catch (Throwable t) {
+ if (t instanceof IOException) {
+ markClosed((IOException)t);
+ } else {
+ markClosed(new IOException("Couldn't set up IO streams: " + t, t));
+ }
+ close();
+ } finally {
+ connectingThread.set(null);
+ }
+ }
+
+ private void setFallBackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth)
+ throws AccessControlException {
+ if (authMethod == null || authProtocol != AuthProtocol.SASL) {
+ if (authProtocol == AuthProtocol.SASL) {
+ LOG.trace("Auth method is not set, yield from setting auth fallback.");
+ }
+ return;
+ }
+ if (fallbackToSimpleAuth == null) {
+ // this should happen only during testing.
+ LOG.trace("Connection {} will skip to set fallbackToSimpleAuth as it is null.", remoteId);
+ } else {
+ if (fallbackToSimpleAuth.get()) {
+ // we already set the value to true, we do not need to examine again.
+ return;
+ }
+ }
+ if (authMethod != AuthMethod.SIMPLE) {
+ if (fallbackToSimpleAuth != null) {
+ LOG.trace("Disabling fallbackToSimpleAuth, target does not use SIMPLE authentication.");
+ fallbackToSimpleAuth.set(false);
+ }
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ if (!fallbackAllowed) {
+ throw new AccessControlException("Server asks us to fall back to SIMPLE auth, but this "
+ + "client is configured to only allow secure connections.");
+ }
+ if (fallbackToSimpleAuth != null) {
+ LOG.trace("Enabling fallbackToSimpleAuth for target, as we are allowed to fall back.");
+ fallbackToSimpleAuth.set(true);
+ }
+ }
+ }
+
+ private void closeConnection() {
+ if (socket == null) {
+ return;
+ }
+ // close the current connection
+ try {
+ socket.close();
+ } catch (IOException e) {
+ LOG.warn("Not able to close a socket", e);
+ }
+ // set socket to null so that the next call to setupIOstreams
+ // can start the process of connect all over again.
+ socket = null;
+ }
+
+ /* Handle connection failures due to timeout on connect
+ *
+ * If the current number of retries is equal to the max number of retries,
+ * stop retrying and throw the exception; Otherwise backoff 1 second and
+ * try connecting again.
+ *
+ * This Method is only called from inside setupIOstreams(), which is
+ * synchronized. Hence the sleep is synchronized; the locks will be retained.
+ *
+ * @param curRetries current number of retries
+ * @param maxRetries max number of retries allowed
+ * @param ioe failure reason
+ * @throws IOException if max number of retries is reached
+ */
+ private void handleConnectionTimeout(
+ int curRetries, int maxRetries, IOException ioe) throws IOException {
+
+ closeConnection();
+
+ // throw the exception if the maximum number of retries is reached
+ if (curRetries >= maxRetries) {
+ throw ioe;
+ }
+ LOG.info("Retrying connect to server: " + server + ". Already tried "
+ + curRetries + " time(s); maxRetries=" + maxRetries);
+ }
+
+ private void handleConnectionFailure(int curRetries, IOException ioe
+ ) throws IOException {
+ closeConnection();
+
+ final RetryAction action;
+ try {
+ action = connectionRetryPolicy.shouldRetry(ioe, curRetries, 0, true);
+ } catch(Exception e) {
+ throw e instanceof IOException? (IOException)e: new IOException(e);
+ }
+ if (action.action == RetryAction.RetryDecision.FAIL) {
+ if (action.reason != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to connect to server: " + server + ": "
+ + action.reason, ioe);
+ }
+ }
+ throw ioe;
+ }
+
+ // Throw the exception if the thread is interrupted
+ if (Thread.currentThread().isInterrupted()) {
+ LOG.warn("Interrupted while trying for connection");
+ throw ioe;
+ }
+
+ try {
+ Thread.sleep(action.delayMillis);
+ } catch (InterruptedException e) {
+ throw (IOException)new InterruptedIOException("Interrupted: action="
+ + action + ", retry policy=" + connectionRetryPolicy).initCause(e);
+ }
+ LOG.info("Retrying connect to server: " + server + ". Already tried "
+ + curRetries + " time(s); retry policy is " + connectionRetryPolicy);
+ }
+
+ /**
+ * Write the connection header - this is sent when connection is established
+ * +----------------------------------+
+ * | "hrpc" 4 bytes |
+ * +----------------------------------+
+ * | Version (1 byte) |
+ * +----------------------------------+
+ * | Service Class (1 byte) |
+ * +----------------------------------+
+ * | AuthProtocol (1 byte) |
+ * +----------------------------------+
+ */
+ private void writeConnectionHeader(IpcStreams streams)
+ throws IOException {
+ // Write out the header, version and authentication method.
+ // The output stream is buffered but we must not flush it yet. The
+ // connection setup protocol requires the client to send multiple
+ // messages before reading a response.
+ //
+ // insecure: send header+context+call, read
+ // secure : send header+negotiate, read, (sasl), context+call, read
+ //
+ // The client must flush only when it's prepared to read. Otherwise
+ // "broken pipe" exceptions occur if the server closes the connection
+ // before all messages are sent.
+ final DataOutputStream out = streams.out;
+ synchronized (out) {
+ out.write(RpcConstants.HEADER.array());
+ out.write(RpcConstants.CURRENT_VERSION);
+ out.write(serviceClass);
+ out.write(authProtocol.callId);
+ }
+ }
+
+ /* Write the connection context header for each connection
+ * Out is not synchronized because only the first thread does this.
+ */
+ private void writeConnectionContext(ConnectionId remoteId,
+ AuthMethod authMethod)
+ throws IOException {
+ // Write out the ConnectionHeader
+ IpcConnectionContextProto message = ProtoUtil.makeIpcConnectionContext(
+ RPC.getProtocolName(remoteId.getProtocol()),
+ remoteId.getTicket(),
+ authMethod);
+ RpcRequestHeaderProto connectionContextHeader = ProtoUtil
+ .makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
+ OperationProto.RPC_FINAL_PACKET, CONNECTION_CONTEXT_CALL_ID,
+ RpcConstants.INVALID_RETRY_COUNT, clientId);
+ // do not flush. the context and first ipc call request must be sent
+ // together to avoid possibility of broken pipes upon authz failure.
+ // see writeConnectionHeader
+ final ResponseBuffer buf = new ResponseBuffer();
+ connectionContextHeader.writeDelimitedTo(buf);
+ message.writeDelimitedTo(buf);
+ synchronized (ipcStreams.out) {
+ ipcStreams.sendRequest(buf.toByteArray());
+ }
+ }
+
+ /* wait till someone signals us to start reading RPC response or
+ * it is idle too long, it is marked as to be closed,
+ * or the client is marked as not running.
+ *
+ * Return true if it is time to read a response; false otherwise.
+ */
+ private synchronized boolean waitForWork() {
+ if (calls.isEmpty() && !shouldCloseConnection.get() && running.get()) {
+ long timeout = maxIdleTime-
+ (Time.now()-lastActivity.get());
+ if (timeout>0) {
+ try {
+ wait(timeout);
+ } catch (InterruptedException e) {
+ LOG.trace("Interrupted while waiting to retrieve RPC response.");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ if (!calls.isEmpty() && !shouldCloseConnection.get() && running.get()) {
+ return true;
+ } else if (shouldCloseConnection.get()) {
+ return false;
+ } else if (calls.isEmpty()) { // idle connection closed or stopped
+ markClosed(null);
+ return false;
+ } else { // get stopped but there are still pending requests
+ markClosed((IOException)new IOException().initCause(
+ new InterruptedException()));
+ return false;
+ }
+ }
+
+ public InetSocketAddress getRemoteAddress() {
+ return server;
+ }
+
+ /* Send a ping to the server if the time elapsed
+ * since last I/O activity is equal to or greater than the ping interval
+ */
+ private synchronized void sendPing() throws IOException {
+ long curTime = Time.now();
+ if ( curTime - lastActivity.get() >= pingInterval) {
+ lastActivity.set(curTime);
+ synchronized (ipcStreams.out) {
+ ipcStreams.sendRequest(pingRequest);
+ ipcStreams.flush();
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ // Don't start the ipc parameter sending thread until we start this
+ // thread, because the shutdown logic only gets triggered if this
+ // thread is started.
+ rpcRequestThread.start();
+ if (LOG.isDebugEnabled())
+ LOG.debug(getName() + ": starting, having connections "
+ + connections.size());
+
+ try {
+ while (waitForWork()) {//wait here for work - read or close connection
+ receiveRpcResponse();
+ }
+ } catch (Throwable t) {
+ // This truly is unexpected, since we catch IOException in receiveResponse
+ // -- this is only to be really sure that we don't leave a client hanging
+ // forever.
+ LOG.warn("Unexpected error reading responses on connection " + this, t);
+ markClosed(new IOException("Error reading responses", t));
+ }
+
+ close();
+
+ if (LOG.isDebugEnabled())
+ LOG.debug(getName() + ": stopped, remaining connections "
+ + connections.size());
+ }
+
+ /**
+ * A thread to write rpc requests to the socket.
+ */
+ private class RpcRequestSender implements Runnable {
+ @Override
+ public void run() {
+ while (!shouldCloseConnection.get()) {
+ ResponseBuffer buf = null;
+ try {
+ Pair pair =
+ rpcRequestQueue.poll(maxIdleTime, TimeUnit.MILLISECONDS);
+ if (pair == null || shouldCloseConnection.get()) {
+ continue;
+ }
+ buf = pair.getRight();
+ synchronized (ipcStreams.out) {
+ if (LOG.isDebugEnabled()) {
+ Call call = pair.getLeft();
+ LOG.debug(getName() + "{} sending #{} {}", getName(), call.id,
+ call.rpcRequest);
+ }
+ // RpcRequestHeader + RpcRequest
+ ipcStreams.sendRequest(buf.toByteArray());
+ ipcStreams.flush();
+ }
+ } catch (InterruptedException ie) {
+ // stop this thread
+ return;
+ } catch (IOException e) {
+ // exception at this point would leave the connection in an
+ // unrecoverable state (eg half a call left on the wire).
+ // So, close the connection, killing any outstanding calls
+ markClosed(e);
+ } finally {
+ //the buffer is just an in-memory buffer, but it is still polite to
+ // close early
+ IOUtils.closeStream(buf);
+ }
+ }
+ }
+ }
+
+ /** Initiates a rpc call by sending the rpc request to the remote server.
+ * Note: this is not called from the current thread, but by another
+ * thread, so that if the current thread is interrupted that the socket
+ * state isn't corrupted with a partially written message.
+ * @param call - the rpc request
+ */
+ public void sendRpcRequest(final Call call)
+ throws InterruptedException, IOException {
+ if (shouldCloseConnection.get()) {
+ return;
+ }
+
+ // Serialize the call to be sent. This is done from the actual
+ // caller thread, rather than the rpcRequestThread in the connection,
+ // so that if the serialization throws an error, it is reported
+ // properly. This also parallelizes the serialization.
+ //
+ // Format of a call on the wire:
+ // 0) Length of rest below (1 + 2)
+ // 1) RpcRequestHeader - is serialized Delimited hence contains length
+ // 2) RpcRequest
+ //
+ // Items '1' and '2' are prepared here.
+ RpcRequestHeaderProto header = ProtoUtil.makeRpcRequestHeader(
+ call.rpcKind, OperationProto.RPC_FINAL_PACKET, call.id, call.retry,
+ clientId, call.alignmentContext);
+
+ final ResponseBuffer buf = new ResponseBuffer();
+ header.writeDelimitedTo(buf);
+ RpcWritable.wrap(call.rpcRequest).writeTo(buf);
+ // Wait for the message to be sent. We offer with timeout to
+ // prevent a race condition between checking the shouldCloseConnection
+ // and the stopping of the polling thread
+ while (!shouldCloseConnection.get()) {
+ if (rpcRequestQueue.offer(Pair.of(call, buf), 1, TimeUnit.SECONDS)) {
+ break;
+ }
+ }
+ }
+
+ /* Receive a response.
+ * Because only one receiver, so no synchronization on in.
+ */
+ private void receiveRpcResponse() {
+ if (shouldCloseConnection.get()) {
+ return;
+ }
+ touch();
+
+ try {
+ ByteBuffer bb = ipcStreams.readResponse();
+ RpcWritable.Buffer packet = RpcWritable.Buffer.wrap(bb);
+ RpcResponseHeaderProto header =
+ packet.getValue(RpcResponseHeaderProto.getDefaultInstance());
+ checkResponse(header);
+
+ int callId = header.getCallId();
+ if (LOG.isDebugEnabled())
+ LOG.debug(getName() + " got value #" + callId);
+
+ RpcStatusProto status = header.getStatus();
+ if (status == RpcStatusProto.SUCCESS) {
+ Writable value = packet.newInstance(valueClass, conf);
+ final Call call = calls.remove(callId);
+ if (call.alignmentContext != null) {
+ call.alignmentContext.receiveResponseState(header);
+ }
+ call.setRpcResponse(value);
+ }
+ // verify that packet length was correct
+ if (packet.remaining() > 0) {
+ throw new RpcClientException("RPC response length mismatch");
+ }
+ if (status != RpcStatusProto.SUCCESS) { // Rpc Request failed
+ final String exceptionClassName = header.hasExceptionClassName() ?
+ header.getExceptionClassName() :
+ "ServerDidNotSetExceptionClassName";
+ final String errorMsg = header.hasErrorMsg() ?
+ header.getErrorMsg() : "ServerDidNotSetErrorMsg" ;
+ final RpcErrorCodeProto erCode =
+ (header.hasErrorDetail() ? header.getErrorDetail() : null);
+ if (erCode == null) {
+ LOG.warn("Detailed error code not set by server on rpc error");
+ }
+ RemoteException re = new RemoteException(exceptionClassName, errorMsg, erCode);
+ if (status == RpcStatusProto.ERROR) {
+ final Call call = calls.remove(callId);
+ call.setException(re);
+ } else if (status == RpcStatusProto.FATAL) {
+ // Close the connection
+ markClosed(re);
+ }
+ }
+ } catch (IOException e) {
+ markClosed(e);
+ }
+ }
+
+ private synchronized void markClosed(IOException e) {
+ if (shouldCloseConnection.compareAndSet(false, true)) {
+ closeException = e;
+ notifyAll();
+ }
+ }
+
+ private void interruptConnectingThread() {
+ Thread connThread = connectingThread.get();
+ if (connThread != null) {
+ connThread.interrupt();
+ }
+ }
+
+ /** Close the connection. */
+ private synchronized void close() {
+ if (!shouldCloseConnection.get()) {
+ LOG.error("The connection is not in the closed state");
+ return;
+ }
+
+ // We have marked this connection as closed. Other thread could have
+ // already known it and replace this closedConnection with a new one.
+ // We should only remove this closedConnection.
+ removeMethod.accept(this);
+
+ // close the streams and therefore the socket
+ IOUtils.closeStream(ipcStreams);
+ disposeSasl();
+
+ // clean up all calls
+ if (closeException == null) {
+ if (!calls.isEmpty()) {
+ LOG.warn(
+ "A connection is closed for no cause and calls are not empty");
+
+ // clean up calls anyway
+ closeException = new IOException("Unexpected closed connection");
+ cleanupCalls();
+ }
+ } else {
+ // Log the newest server information if update address.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("closing ipc connection to " + server + ": " +
+ closeException.getMessage(),closeException);
+ }
+
+ // cleanup calls
+ cleanupCalls();
+ }
+ closeConnection();
+ if (LOG.isDebugEnabled())
+ LOG.debug(getName() + ": closed");
+ }
+
+ /* Cleanup all calls and mark them as done */
+ private void cleanupCalls() {
+ Iterator> itor = calls.entrySet().iterator() ;
+ while (itor.hasNext()) {
+ Call c = itor.next().getValue();
+ itor.remove();
+ c.setException(closeException); // local exception
+ }
+ }
+ }
+
+ /**
+ * Construct an IPC client whose values are of the given {@link Writable}
+ * class.
+ *
+ * @param valueClass input valueClass.
+ * @param conf input configuration.
+ * @param factory input factory.
+ */
+ public Client(Class valueClass, Configuration conf,
+ SocketFactory factory) {
+ this.valueClass = valueClass;
+ this.conf = conf;
+ this.socketFactory = factory;
+ this.connectionTimeout = conf.getInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT);
+ this.fallbackAllowed = conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
+ this.bindToWildCardAddress = conf
+ .getBoolean(CommonConfigurationKeys.IPC_CLIENT_BIND_WILDCARD_ADDR_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_BIND_WILDCARD_ADDR_DEFAULT);
+
+ this.clientId = ClientId.getClientId();
+ this.maxAsyncCalls = conf.getInt(
+ CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_DEFAULT);
+ }
+
+ /**
+ * Construct an IPC client with the default SocketFactory.
+ * @param valueClass input valueClass.
+ * @param conf input Configuration.
+ */
+ public Client(Class valueClass, Configuration conf) {
+ this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "-"
+ + StringUtils.byteToHexString(clientId);
+ }
+
+ /** Return the socket factory of this client
+ *
+ * @return this client's socket factory
+ */
+ SocketFactory getSocketFactory() {
+ return socketFactory;
+ }
+
+ /** Stop all threads related to this client. No further calls may be made
+ * using this client. */
+ public void stop() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping client");
+ }
+ synchronized (putLock) { // synchronized to avoid put after stop
+ if (!running.compareAndSet(true, false)) {
+ return;
+ }
+ }
+
+ // wake up all connections
+ for (Connection conn : connections.values()) {
+ conn.interrupt();
+ conn.rpcRequestThread.interrupt();
+ conn.interruptConnectingThread();
+ }
+
+ // wait until all connections are closed
+ synchronized (emptyCondition) {
+ // synchronized the loop to guarantee wait must be notified.
+ while (!connections.isEmpty()) {
+ try {
+ emptyCondition.wait();
+ } catch (InterruptedException e) {
+ LOG.trace(
+ "Interrupted while waiting on all connections to be closed.");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+
+ /**
+ * Make a call, passing rpcRequest, to the IPC server defined by
+ * remoteId, returning the rpc respond.
+ *
+ * @param rpcKind - input rpcKind.
+ * @param rpcRequest - contains serialized method and method parameters
+ * @param remoteId - the target rpc server
+ * @param fallbackToSimpleAuth - set to true or false during this method to
+ * indicate if a secure client falls back to simple auth
+ * @return the rpc response
+ * Throws exceptions if there are network problems or if the remote code
+ * threw an exception.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+ ConnectionId remoteId, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
+ return call(rpcKind, rpcRequest, remoteId, RPC.RPC_SERVICE_CLASS_DEFAULT,
+ fallbackToSimpleAuth, null);
+ }
+
+ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+ ConnectionId remoteId, AtomicBoolean fallbackToSimpleAuth,
+ AlignmentContext alignmentContext)
+ throws IOException {
+ return call(rpcKind, rpcRequest, remoteId, RPC.RPC_SERVICE_CLASS_DEFAULT,
+ fallbackToSimpleAuth, alignmentContext);
+ }
+
+ private void checkAsyncCall() throws IOException {
+ if (isAsynchronousMode()) {
+ if (asyncCallCounter.incrementAndGet() > maxAsyncCalls) {
+ asyncCallCounter.decrementAndGet();
+ String errMsg = String.format(
+ "Exceeded limit of max asynchronous calls: %d, " +
+ "please configure %s to adjust it.",
+ maxAsyncCalls,
+ CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_KEY);
+ throw new AsyncCallLimitExceededException(errMsg);
+ }
+ }
+ }
+
+ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+ ConnectionId remoteId, int serviceClass,
+ AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
+ return call(rpcKind, rpcRequest, remoteId, serviceClass,
+ fallbackToSimpleAuth, null);
+ }
+
+ /**
+ * Make a call, passing rpcRequest, to the IPC server defined by
+ * remoteId, returning the rpc response.
+ *
+ * @param rpcKind
+ * @param rpcRequest - contains serialized method and method parameters
+ * @param remoteId - the target rpc server
+ * @param serviceClass - service class for RPC
+ * @param fallbackToSimpleAuth - set to true or false during this method to
+ * indicate if a secure client falls back to simple auth
+ * @param alignmentContext - state alignment context
+ * @return the rpc response
+ * Throws exceptions if there are network problems or if the remote code
+ * threw an exception.
+ */
+ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+ ConnectionId remoteId, int serviceClass,
+ AtomicBoolean fallbackToSimpleAuth, AlignmentContext alignmentContext)
+ throws IOException {
+ final Call call = createCall(rpcKind, rpcRequest);
+ call.setAlignmentContext(alignmentContext);
+ final Connection connection = getConnection(remoteId, call, serviceClass,
+ fallbackToSimpleAuth);
+
+ try {
+ checkAsyncCall();
+ try {
+ connection.sendRpcRequest(call); // send the rpc request
+ } catch (RejectedExecutionException e) {
+ throw new IOException("connection has been closed", e);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ IOException ioe = new InterruptedIOException(
+ "Interrupted waiting to send RPC request to server");
+ ioe.initCause(ie);
+ throw ioe;
+ }
+ } catch(Exception e) {
+ if (isAsynchronousMode()) {
+ releaseAsyncCall();
+ }
+ throw e;
+ }
+
+ if (isAsynchronousMode()) {
+ final AsyncGet asyncGet
+ = new AsyncGet() {
+ @Override
+ public Writable get(long timeout, TimeUnit unit)
+ throws IOException, TimeoutException{
+ boolean done = true;
+ try {
+ final Writable w = getRpcResponse(call, connection, timeout, unit);
+ if (w == null) {
+ done = false;
+ throw new TimeoutException(call + " timed out "
+ + timeout + " " + unit);
+ }
+ return w;
+ } finally {
+ if (done) {
+ releaseAsyncCall();
+ }
+ }
+ }
+
+ @Override
+ public boolean isDone() {
+ synchronized (call) {
+ return call.done;
+ }
+ }
+ };
+
+ ASYNC_RPC_RESPONSE.set(asyncGet);
+ return null;
+ } else {
+ return getRpcResponse(call, connection, -1, null);
+ }
+ }
+
+ /**
+ * Check if RPC is in asynchronous mode or not.
+ *
+ * @return true, if RPC is in asynchronous mode, otherwise false for
+ * synchronous mode.
+ */
+ public static boolean isAsynchronousMode() {
+ return asynchronousMode.get();
+ }
+
+ /**
+ * Set RPC to asynchronous or synchronous mode.
+ *
+ * @param async
+ * true, RPC will be in asynchronous mode, otherwise false for
+ * synchronous mode
+ */
+ public static void setAsynchronousMode(boolean async) {
+ asynchronousMode.set(async);
+ }
+
+ private void releaseAsyncCall() {
+ asyncCallCounter.decrementAndGet();
+ }
+
+ int getAsyncCallCount() {
+ return asyncCallCounter.get();
+ }
+
+ /** @return the rpc response or, in case of timeout, null. */
+ private Writable getRpcResponse(final Call call, final Connection connection,
+ final long timeout, final TimeUnit unit) throws IOException {
+ synchronized (call) {
+ while (!call.done) {
+ try {
+ AsyncGet.Util.wait(call, timeout, unit);
+ if (timeout >= 0 && !call.done) {
+ return null;
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedIOException("Call interrupted");
+ }
+ }
+
+ if (call.error != null) {
+ if (call.error instanceof RemoteException) {
+ call.error.fillInStackTrace();
+ throw call.error;
+ } else { // local exception
+ InetSocketAddress address = connection.getRemoteAddress();
+ throw NetUtils.wrapException(address.getHostName(),
+ address.getPort(),
+ NetUtils.getHostname(),
+ 0,
+ call.error);
+ }
+ } else {
+ return call.getRpcResponse();
+ }
+ }
+ }
+
+ // for unit testing only
+ Set getConnectionIds() {
+ return connections.keySet();
+ }
+
+ /** Get a connection from the pool, or create a new one and add it to the
+ * pool. Connections to a given ConnectionId are reused. */
+ private Connection getConnection(ConnectionId remoteId,
+ Call call, int serviceClass, AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
+ final Consumer removeMethod = c -> {
+ final boolean removed = connections.remove(remoteId, c);
+ if (removed && connections.isEmpty()) {
+ synchronized (emptyCondition) {
+ emptyCondition.notify();
+ }
+ }
+ };
+
+ Connection connection;
+ /* we could avoid this allocation for each RPC by having a
+ * connectionsId object and with set() method. We need to manage the
+ * refs for keys in HashMap properly. For now its ok.
+ */
+ while (true) {
+ synchronized (putLock) { // synchronized to avoid put after stop
+ if (!running.get()) {
+ throw new IOException("Failed to get connection for " + remoteId
+ + ", " + call + ": " + this + " is already stopped");
+ }
+ connection = connections.computeIfAbsent(remoteId,
+ id -> new Connection(id, serviceClass, removeMethod));
+ }
+
+ if (connection.addCall(call)) {
+ break;
+ } else {
+ // This connection is closed, should be removed. But other thread could
+ // have already known this closedConnection, and replace it with a new
+ // connection. So we should call conditional remove to make sure we only
+ // remove this closedConnection.
+ removeMethod.accept(connection);
+ }
+ }
+
+ // If the server happens to be slow, the method below will take longer to
+ // establish a connection.
+ connection.setupIOstreams(fallbackToSimpleAuth);
+ return connection;
+ }
+
+ /**
+ * This class holds the address and the user ticket. The client connections
+ * to servers are uniquely identified by [remoteAddress, protocol, ticket]
+ */
+ public static class ConnectionId {
+ private InetSocketAddress address;
+ private final UserGroupInformation ticket;
+ private final Class protocol;
+ private static final int PRIME = 16777619;
+ private final int rpcTimeout;
+ private final int maxIdleTime; //connections will be culled if it was idle for
+ //maxIdleTime msecs
+ private final RetryPolicy connectionRetryPolicy;
+ private final int maxRetriesOnSasl;
+ // the max. no. of retries for socket connections on time out exceptions
+ private final int maxRetriesOnSocketTimeouts;
+ private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+ private final boolean tcpLowLatency; // if T then use low-delay QoS
+ private final boolean doPing; //do we need to send ping message
+ private final int pingInterval; // how often sends ping to the server in msecs
+ private String saslQop; // here for testing
+ private final Configuration conf; // used to get the expected kerberos principal name
+
+ public ConnectionId(InetSocketAddress address, Class protocol,
+ UserGroupInformation ticket, int rpcTimeout,
+ RetryPolicy connectionRetryPolicy, Configuration conf) {
+ this.protocol = protocol;
+ this.address = address;
+ this.ticket = ticket;
+ this.rpcTimeout = rpcTimeout;
+ this.connectionRetryPolicy = connectionRetryPolicy;
+
+ this.maxIdleTime = conf.getInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT);
+ this.maxRetriesOnSasl = conf.getInt(
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_DEFAULT);
+ this.maxRetriesOnSocketTimeouts = conf.getInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT);
+ this.tcpNoDelay = conf.getBoolean(
+ CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY,
+ CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_DEFAULT);
+ this.tcpLowLatency = conf.getBoolean(
+ CommonConfigurationKeysPublic.IPC_CLIENT_LOW_LATENCY,
+ CommonConfigurationKeysPublic.IPC_CLIENT_LOW_LATENCY_DEFAULT
+ );
+ this.doPing = conf.getBoolean(
+ CommonConfigurationKeys.IPC_CLIENT_PING_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT);
+ this.pingInterval = (doPing ? Client.getPingInterval(conf) : 0);
+ this.conf = conf;
+ }
+
+ InetSocketAddress getAddress() {
+ return address;
+ }
+
+ /**
+ * This is used to update the remote address when an address change is detected. This method
+ * ensures that the {@link #hashCode()} won't change.
+ *
+ * @param address the updated address
+ * @throws IllegalArgumentException if the hostname or port doesn't match
+ * @see Connection#updateAddress()
+ */
+ void setAddress(InetSocketAddress address) {
+ if (!Objects.equals(this.address.getHostName(), address.getHostName())) {
+ throw new IllegalArgumentException("Hostname must match: " + this.address + " vs "
+ + address);
+ }
+ if (this.address.getPort() != address.getPort()) {
+ throw new IllegalArgumentException("Port must match: " + this.address + " vs " + address);
+ }
+
+ this.address = address;
+ }
+
+
+ Class getProtocol() {
+ return protocol;
+ }
+
+ UserGroupInformation getTicket() {
+ return ticket;
+ }
+
+ int getRpcTimeout() {
+ return rpcTimeout;
+ }
+
+ int getMaxIdleTime() {
+ return maxIdleTime;
+ }
+
+ public int getMaxRetriesOnSasl() {
+ return maxRetriesOnSasl;
+ }
+
+ /** @return max connection retries on socket time outs */
+ public int getMaxRetriesOnSocketTimeouts() {
+ return maxRetriesOnSocketTimeouts;
+ }
+
+ /** disable nagle's algorithm */
+ boolean getTcpNoDelay() {
+ return tcpNoDelay;
+ }
+
+ /** use low-latency QoS bits over TCP */
+ boolean getTcpLowLatency() {
+ return tcpLowLatency;
+ }
+
+ boolean getDoPing() {
+ return doPing;
+ }
+
+ int getPingInterval() {
+ return pingInterval;
+ }
+
+ RetryPolicy getRetryPolicy() {
+ return connectionRetryPolicy;
+ }
+
+ String getSaslQop() {
+ return saslQop;
+ }
+
+ /**
+ * Returns a ConnectionId object.
+ * @param addr Remote address for the connection.
+ * @param protocol Protocol for RPC.
+ * @param ticket UGI
+ * @param rpcTimeout timeout
+ * @param conf Configuration object
+ * @return A ConnectionId instance
+ * @throws IOException
+ */
+ static ConnectionId getConnectionId(InetSocketAddress addr,
+ Class protocol, UserGroupInformation ticket, int rpcTimeout,
+ RetryPolicy connectionRetryPolicy, Configuration conf) throws IOException {
+
+ if (connectionRetryPolicy == null) {
+ final int max = conf.getInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT);
+ final int retryInterval = conf.getInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY,
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT);
+
+ connectionRetryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
+ max, retryInterval, TimeUnit.MILLISECONDS);
+ }
+
+ return new ConnectionId(addr, protocol, ticket, rpcTimeout,
+ connectionRetryPolicy, conf);
+ }
+
+ static boolean isEqual(Object a, Object b) {
+ return a == null ? b == null : a.equals(b);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (obj instanceof ConnectionId) {
+ ConnectionId that = (ConnectionId) obj;
+ return isEqual(this.address, that.address)
+ && this.doPing == that.doPing
+ && this.maxIdleTime == that.maxIdleTime
+ && isEqual(this.connectionRetryPolicy, that.connectionRetryPolicy)
+ && this.pingInterval == that.pingInterval
+ && isEqual(this.protocol, that.protocol)
+ && this.rpcTimeout == that.rpcTimeout
+ && this.tcpNoDelay == that.tcpNoDelay
+ && isEqual(this.ticket, that.ticket);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = connectionRetryPolicy.hashCode();
+ // We calculate based on the host name and port without the IP address, since the hashCode
+ // must be stable even if the IP address is updated.
+ result = PRIME * result + ((address == null || address.getHostName() == null) ? 0 :
+ address.getHostName().hashCode());
+ result = PRIME * result + ((address == null) ? 0 : address.getPort());
+ result = PRIME * result + (doPing ? 1231 : 1237);
+ result = PRIME * result + maxIdleTime;
+ result = PRIME * result + pingInterval;
+ result = PRIME * result + ((protocol == null) ? 0 : protocol.hashCode());
+ result = PRIME * result + rpcTimeout;
+ result = PRIME * result + (tcpNoDelay ? 1231 : 1237);
+ result = PRIME * result + ((ticket == null) ? 0 : ticket.hashCode());
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return address.toString();
+ }
+ }
+
+ /**
+ * Returns the next valid sequential call ID by incrementing an atomic counter
+ * and masking off the sign bit. Valid call IDs are non-negative integers in
+ * the range [ 0, 2^31 - 1 ]. Negative numbers are reserved for special
+ * purposes. The values can overflow back to 0 and be reused. Note that prior
+ * versions of the client did not mask off the sign bit, so a server may still
+ * see a negative call ID if it receives connections from an old client.
+ *
+ * @return next call ID
+ */
+ public static int nextCallId() {
+ return callIdCounter.getAndIncrement() & 0x7FFFFFFF;
+ }
+
+ @Override
+ public void close() throws Exception {
+ stop();
+ }
+
+ /** Manages the input and output streams for an IPC connection.
+ * Only exposed for use by SaslRpcClient.
+ */
+ public static class IpcStreams implements Closeable, Flushable {
+ private DataInputStream in;
+ public DataOutputStream out;
+ private int maxResponseLength;
+ private boolean firstResponse = true;
+
+ IpcStreams(Socket socket, int maxResponseLength) throws IOException {
+ this.maxResponseLength = maxResponseLength;
+ setInputStream(
+ new BufferedInputStream(NetUtils.getInputStream(socket)));
+ setOutputStream(
+ new BufferedOutputStream(NetUtils.getOutputStream(socket)));
+ }
+
+ void setSaslClient(SaslRpcClient client) throws IOException {
+ // Wrap the input stream in a BufferedInputStream to fill the buffer
+ // before reading its length (HADOOP-14062).
+ setInputStream(new BufferedInputStream(client.getInputStream(in)));
+ setOutputStream(client.getOutputStream(out));
+ }
+
+ private void setInputStream(InputStream is) {
+ this.in = (is instanceof DataInputStream)
+ ? (DataInputStream)is : new DataInputStream(is);
+ }
+
+ private void setOutputStream(OutputStream os) {
+ this.out = (os instanceof DataOutputStream)
+ ? (DataOutputStream)os : new DataOutputStream(os);
+ }
+
+ public ByteBuffer readResponse() throws IOException {
+ int length = in.readInt();
+ if (firstResponse) {
+ firstResponse = false;
+ // pre-rpcv9 exception, almost certainly a version mismatch.
+ if (length == -1) {
+ in.readInt(); // ignore fatal/error status, it's fatal for us.
+ throw new RemoteException(WritableUtils.readString(in),
+ WritableUtils.readString(in));
+ }
+ }
+ if (length <= 0) {
+ throw new RpcException(String.format("RPC response has " +
+ "invalid length of %d", length));
+ }
+ if (maxResponseLength > 0 && length > maxResponseLength) {
+ throw new RpcException(String.format("RPC response has a " +
+ "length of %d exceeds maximum data length", length));
+ }
+ ByteBuffer bb = ByteBuffer.allocate(length);
+ in.readFully(bb.array());
+ return bb;
+ }
+
+ public void sendRequest(byte[] buf) throws IOException {
+ out.write(buf);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeStream(out);
+ IOUtils.closeStream(in);
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ClientCache.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ClientCache.java
new file mode 100644
index 000000000000..93c2aa6ec186
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ClientCache.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.io.Writable;
+
+
+/* Cache a client using its socket factory as the hash key */
+public class ClientCache {
+ private Map clients =
+ new HashMap();
+
+ /**
+ * Construct and cache an IPC client with the user-provided SocketFactory
+ * if no cached client exists.
+ *
+ * @param conf Configuration
+ * @param factory SocketFactory for client socket
+ * @param valueClass Class of the expected response
+ * @return an IPC client
+ */
+ public synchronized Client getClient(Configuration conf,
+ SocketFactory factory, Class valueClass) {
+ // Construct & cache client. The configuration is only used for timeout,
+ // and Clients have connection pools. So we can either (a) lose some
+ // connection pooling and leak sockets, or (b) use the same timeout for all
+ // configurations. Since the IPC is usually intended globally, not
+ // per-job, we choose (a).
+ Client client = clients.get(factory);
+ if (client == null) {
+ client = new Client(valueClass, conf, factory);
+ clients.put(factory, client);
+ } else {
+ client.incCount();
+ }
+ if (Client.LOG.isDebugEnabled()) {
+ Client.LOG.debug("getting client out of cache: " + client);
+ }
+ return client;
+ }
+
+ /**
+ * Construct and cache an IPC client with the default SocketFactory
+ * and default valueClass if no cached client exists.
+ *
+ * @param conf Configuration
+ * @return an IPC client
+ */
+ public synchronized Client getClient(Configuration conf) {
+ return getClient(conf, SocketFactory.getDefault(), ObjectWritable.class);
+ }
+
+ /**
+ * Construct and cache an IPC client with the user-provided SocketFactory
+ * if no cached client exists. Default response type is ObjectWritable.
+ *
+ * @param conf Configuration
+ * @param factory SocketFactory for client socket
+ * @return an IPC client
+ */
+ public synchronized Client getClient(Configuration conf, SocketFactory factory) {
+ return this.getClient(conf, factory, ObjectWritable.class);
+ }
+
+ /**
+ * Stop a RPC client connection
+ * A RPC client is closed only when its reference count becomes zero.
+ *
+ * @param client input client.
+ */
+ public void stopClient(Client client) {
+ if (Client.LOG.isDebugEnabled()) {
+ Client.LOG.debug("stopping client from cache: " + client);
+ }
+ final int count;
+ synchronized (this) {
+ count = client.decAndGetCount();
+ if (count == 0) {
+ if (Client.LOG.isDebugEnabled()) {
+ Client.LOG.debug("removing client from cache: " + client);
+ }
+ clients.remove(client.getSocketFactory());
+ }
+ }
+ if (count == 0) {
+ if (Client.LOG.isDebugEnabled()) {
+ Client.LOG.debug("stopping actual client because no more references remain: "
+ + client);
+ }
+ client.stop();
+ }
+ }
+
+ public void clearCache() {
+ clients.values().forEach(c -> c.stop());
+ clients.clear();
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ClientId.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ClientId.java
new file mode 100644
index 000000000000..77d73a27e7aa
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ClientId.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A class defining a set of static helper methods to provide conversion between
+ * bytes and string for UUID-based client Id.
+ */
+public class ClientId {
+
+ /** The byte array of a UUID should be 16 */
+ public static final int BYTE_LENGTH = 16;
+ private static final int shiftWidth = 8;
+
+ /**
+ * @return Return clientId as byte[].
+ */
+ public static byte[] getClientId() {
+ UUID uuid = UUID.randomUUID();
+ ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
+ buf.putLong(uuid.getMostSignificantBits());
+ buf.putLong(uuid.getLeastSignificantBits());
+ return buf.array();
+ }
+
+ /**
+ * @return Convert a clientId byte[] to string.
+ * @param clientId input clientId.
+ */
+ public static String toString(byte[] clientId) {
+ // clientId can be null or an empty array
+ if (clientId == null || clientId.length == 0) {
+ return "";
+ }
+ // otherwise should be 16 bytes
+ Preconditions.checkArgument(clientId.length == BYTE_LENGTH);
+ long msb = getMsb(clientId);
+ long lsb = getLsb(clientId);
+ return (new UUID(msb, lsb)).toString();
+ }
+
+ public static long getMsb(byte[] clientId) {
+ long msb = 0;
+ for (int i = 0; i < BYTE_LENGTH/2; i++) {
+ msb = (msb << shiftWidth) | (clientId[i] & 0xff);
+ }
+ return msb;
+ }
+
+ public static long getLsb(byte[] clientId) {
+ long lsb = 0;
+ for (int i = BYTE_LENGTH/2; i < BYTE_LENGTH; i++) {
+ lsb = (lsb << shiftWidth) | (clientId[i] & 0xff);
+ }
+ return lsb;
+ }
+
+ /**
+ * @return Convert from clientId string byte[] representation of clientId.
+ * @param id input id.
+ */
+ public static byte[] toBytes(String id) {
+ if (id == null || "".equals(id)) {
+ return new byte[0];
+ }
+ UUID uuid = UUID.fromString(id);
+ ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
+ buf.putLong(uuid.getMostSignificantBits());
+ buf.putLong(uuid.getLeastSignificantBits());
+ return buf.array();
+ }
+
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/CostProvider.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/CostProvider.java
new file mode 100644
index 000000000000..552b50bb17bc
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/CostProvider.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Used by {@link DecayRpcScheduler} to get the cost of users' operations. This
+ * is configurable using
+ * {@link org.apache.hadoop.fs.CommonConfigurationKeys#IPC_COST_PROVIDER_KEY}.
+ */
+public interface CostProvider {
+
+ /**
+ * Initialize this provider using the given configuration, examining only
+ * ones which fall within the provided namespace.
+ *
+ * @param namespace The namespace to use when looking up configurations.
+ * @param conf The configuration
+ */
+ void init(String namespace, Configuration conf);
+
+ /**
+ * Get cost from {@link ProcessingDetails} which will be used in scheduler.
+ *
+ * @param details Process details
+ * @return The cost of the call
+ */
+ long getCost(ProcessingDetails details);
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DecayRpcScheduler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DecayRpcScheduler.java
new file mode 100644
index 000000000000..b0bf705553ab
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DecayRpcScheduler.java
@@ -0,0 +1,1029 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongArray;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AtomicDoubleArray;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ipc_.metrics.RpcMetrics;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair;
+import org.apache.hadoop.metrics2.util.Metrics2Util.TopN;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ipc_.ProcessingDetails.Timing;
+
+/**
+ * The decay RPC scheduler tracks the cost of incoming requests in a map, then
+ * decays the costs at a fixed time interval. The scheduler is optimized
+ * for large periods (on the order of seconds), as it offloads work to the
+ * decay sweep.
+ */
+public class DecayRpcScheduler implements RpcScheduler,
+ DecayRpcSchedulerMXBean, MetricsSource {
+ /**
+ * Period controls how many milliseconds between each decay sweep.
+ */
+ public static final String IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY =
+ "decay-scheduler.period-ms";
+ public static final long IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT =
+ 5000;
+ @Deprecated
+ public static final String IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY =
+ "faircallqueue.decay-scheduler.period-ms";
+
+ /**
+ * Decay factor controls how much each count is suppressed by on each sweep.
+ * Valid numbers are between 0 and 1, exclusive. Decay factor works in tandem
+ * with period to control how long the scheduler remembers an identity.
+ */
+ public static final String IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY =
+ "decay-scheduler.decay-factor";
+ public static final double IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT =
+ 0.5;
+ @Deprecated
+ public static final String IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY =
+ "faircallqueue.decay-scheduler.decay-factor";
+
+ /**
+ * Thresholds are specified as integer percentages, and specify which usage
+ * range each queue will be allocated to. For instance, specifying the list
+ * 10, 40, 80
+ * implies 4 queues, with
+ * - q3 from 80% up
+ * - q2 from 40 up to 80
+ * - q1 from 10 up to 40
+ * - q0 otherwise.
+ */
+ public static final String IPC_DECAYSCHEDULER_THRESHOLDS_KEY =
+ "decay-scheduler.thresholds";
+ @Deprecated
+ public static final String IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY =
+ "faircallqueue.decay-scheduler.thresholds";
+
+ // Specifies the identity to use when the IdentityProvider cannot handle
+ // a schedulable.
+ public static final String DECAYSCHEDULER_UNKNOWN_IDENTITY =
+ "IdentityProvider.Unknown";
+
+ public static final String
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY =
+ "decay-scheduler.backoff.responsetime.enable";
+ public static final Boolean
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_DEFAULT = false;
+
+ // Specifies the average response time (ms) thresholds of each
+ // level to trigger backoff
+ public static final String
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY =
+ "decay-scheduler.backoff.responsetime.thresholds";
+
+ // Specifies the top N user's call count and scheduler decision
+ // Metrics2 Source
+ public static final String DECAYSCHEDULER_METRICS_TOP_USER_COUNT =
+ "decay-scheduler.metrics.top.user.count";
+ public static final int DECAYSCHEDULER_METRICS_TOP_USER_COUNT_DEFAULT = 10;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DecayRpcScheduler.class);
+
+ private static final ObjectWriter WRITER = new ObjectMapper().writer();
+
+ // Track the decayed and raw (no decay) number of calls for each schedulable
+ // identity from all previous decay windows: idx 0 for decayed call cost and
+ // idx 1 for the raw call cost
+ private final ConcurrentHashMap> callCosts =
+ new ConcurrentHashMap>();
+
+ // Should be the sum of all AtomicLongs in decayed callCosts
+ private final AtomicLong totalDecayedCallCost = new AtomicLong();
+ // The sum of all AtomicLongs in raw callCosts
+ private final AtomicLong totalRawCallCost = new AtomicLong();
+
+
+ // Track total call count and response time in current decay window
+ private final AtomicLongArray responseTimeCountInCurrWindow;
+ private final AtomicLongArray responseTimeTotalInCurrWindow;
+
+ // Track average response time in previous decay window
+ private final AtomicDoubleArray responseTimeAvgInLastWindow;
+ private final AtomicLongArray responseTimeCountInLastWindow;
+
+ // Pre-computed scheduling decisions during the decay sweep are
+ // atomically swapped in as a read-only map
+ private final AtomicReference> scheduleCacheRef =
+ new AtomicReference>();
+
+ // Tune the behavior of the scheduler
+ private final long decayPeriodMillis; // How long between each tick
+ private final double decayFactor; // nextCost = currentCost * decayFactor
+ private final int numLevels;
+ private final double[] thresholds;
+ private final IdentityProvider identityProvider;
+ private final boolean backOffByResponseTimeEnabled;
+ private final long[] backOffResponseTimeThresholds;
+ private final String namespace;
+ private final int topUsersCount; // e.g., report top 10 users' metrics
+ private static final double PRECISION = 0.0001;
+ private MetricsProxy metricsProxy;
+ private final CostProvider costProvider;
+
+ private final Map staticPriorities = new HashMap<>();
+ /**
+ * This TimerTask will call decayCurrentCosts until
+ * the scheduler has been garbage collected.
+ */
+ public static class DecayTask extends TimerTask {
+ private WeakReference schedulerRef;
+ private Timer timer;
+
+ public DecayTask(DecayRpcScheduler scheduler, Timer timer) {
+ this.schedulerRef = new WeakReference(scheduler);
+ this.timer = timer;
+ }
+
+ @Override
+ public void run() {
+ DecayRpcScheduler sched = schedulerRef.get();
+ if (sched != null) {
+ sched.decayCurrentCosts();
+ } else {
+ // Our scheduler was garbage collected since it is no longer in use,
+ // so we should terminate the timer as well
+ timer.cancel();
+ timer.purge();
+ }
+ }
+ }
+
+ /**
+ * Create a decay scheduler.
+ * @param numLevels number of priority levels
+ * @param ns config prefix, so that we can configure multiple schedulers
+ * in a single instance.
+ * @param conf configuration to use.
+ */
+ public DecayRpcScheduler(int numLevels, String ns, Configuration conf) {
+ if(numLevels < 1) {
+ throw new IllegalArgumentException("Number of Priority Levels must be " +
+ "at least 1");
+ }
+ this.numLevels = numLevels;
+ this.namespace = ns;
+ this.decayFactor = parseDecayFactor(ns, conf);
+ this.decayPeriodMillis = parseDecayPeriodMillis(ns, conf);
+ this.identityProvider = this.parseIdentityProvider(ns, conf);
+ this.costProvider = this.parseCostProvider(ns, conf);
+ this.thresholds = parseThresholds(ns, conf, numLevels);
+ this.backOffByResponseTimeEnabled = parseBackOffByResponseTimeEnabled(ns,
+ conf);
+ this.backOffResponseTimeThresholds =
+ parseBackOffResponseTimeThreshold(ns, conf, numLevels);
+
+ // Setup response time metrics
+ responseTimeTotalInCurrWindow = new AtomicLongArray(numLevels);
+ responseTimeCountInCurrWindow = new AtomicLongArray(numLevels);
+ responseTimeAvgInLastWindow = new AtomicDoubleArray(numLevels);
+ responseTimeCountInLastWindow = new AtomicLongArray(numLevels);
+
+ topUsersCount =
+ conf.getInt(DECAYSCHEDULER_METRICS_TOP_USER_COUNT,
+ DECAYSCHEDULER_METRICS_TOP_USER_COUNT_DEFAULT);
+ Preconditions.checkArgument(topUsersCount > 0,
+ "the number of top users for scheduler metrics must be at least 1");
+
+ // Setup delay timer
+ Timer timer = new Timer(true);
+ DecayTask task = new DecayTask(this, timer);
+ timer.scheduleAtFixedRate(task, decayPeriodMillis, decayPeriodMillis);
+
+ metricsProxy = MetricsProxy.getInstance(ns, numLevels, this);
+ recomputeScheduleCache();
+ }
+
+ private CostProvider parseCostProvider(String ns, Configuration conf) {
+ List providers = conf.getInstances(
+ ns + "." + CommonConfigurationKeys.IPC_COST_PROVIDER_KEY,
+ CostProvider.class);
+
+ if (providers.size() < 1) {
+ LOG.info("CostProvider not specified, defaulting to DefaultCostProvider");
+ return new DefaultCostProvider();
+ } else if (providers.size() > 1) {
+ LOG.warn("Found multiple CostProviders; using: {}",
+ providers.get(0).getClass());
+ }
+
+ CostProvider provider = providers.get(0); // use the first
+ provider.init(ns, conf);
+ return provider;
+ }
+
+ // Load configs
+ private IdentityProvider parseIdentityProvider(String ns,
+ Configuration conf) {
+ List providers = conf.getInstances(
+ ns + "." + CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY,
+ IdentityProvider.class);
+
+ if (providers.size() < 1) {
+ LOG.info("IdentityProvider not specified, " +
+ "defaulting to UserIdentityProvider");
+ return new UserIdentityProvider();
+ }
+
+ return providers.get(0); // use the first
+ }
+
+ private static double parseDecayFactor(String ns, Configuration conf) {
+ double factor = conf.getDouble(ns + "." +
+ IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, 0.0);
+ if (factor == 0.0) {
+ factor = conf.getDouble(ns + "." +
+ IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY,
+ IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT);
+ } else if ((factor > 0.0) && (factor < 1)) {
+ LOG.warn(IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY +
+ " is deprecated. Please use " +
+ IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY + ".");
+ }
+ if (factor <= 0 || factor >= 1) {
+ throw new IllegalArgumentException("Decay Factor " +
+ "must be between 0 and 1");
+ }
+
+ return factor;
+ }
+
+ private static long parseDecayPeriodMillis(String ns, Configuration conf) {
+ long period = conf.getLong(ns + "." +
+ IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY,
+ 0);
+ if (period == 0) {
+ period = conf.getLong(ns + "." +
+ IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY,
+ IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT);
+ } else if (period > 0) {
+ LOG.warn((IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY +
+ " is deprecated. Please use " +
+ IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY));
+ }
+ if (period <= 0) {
+ throw new IllegalArgumentException("Period millis must be >= 0");
+ }
+
+ return period;
+ }
+
+ private static double[] parseThresholds(String ns, Configuration conf,
+ int numLevels) {
+ int[] percentages = conf.getInts(ns + "." +
+ IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY);
+
+ if (percentages.length == 0) {
+ percentages = conf.getInts(ns + "." + IPC_DECAYSCHEDULER_THRESHOLDS_KEY);
+ if (percentages.length == 0) {
+ return getDefaultThresholds(numLevels);
+ }
+ } else {
+ LOG.warn(IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY +
+ " is deprecated. Please use " +
+ IPC_DECAYSCHEDULER_THRESHOLDS_KEY);
+ }
+
+ if (percentages.length != numLevels-1) {
+ throw new IllegalArgumentException("Number of thresholds should be " +
+ (numLevels-1) + ". Was: " + percentages.length);
+ }
+
+ // Convert integer percentages to decimals
+ double[] decimals = new double[percentages.length];
+ for (int i = 0; i < percentages.length; i++) {
+ decimals[i] = percentages[i] / 100.0;
+ }
+
+ return decimals;
+ }
+
+ /**
+ * Generate default thresholds if user did not specify. Strategy is
+ * to halve each time, since queue usage tends to be exponential.
+ * So if numLevels is 4, we would generate: double[]{0.125, 0.25, 0.5}
+ * which specifies the boundaries between each queue's usage.
+ * @param numLevels number of levels to compute for
+ * @return array of boundaries of length numLevels - 1
+ */
+ private static double[] getDefaultThresholds(int numLevels) {
+ double[] ret = new double[numLevels - 1];
+ double div = Math.pow(2, numLevels - 1);
+
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = Math.pow(2, i)/div;
+ }
+ return ret;
+ }
+
+ private static long[] parseBackOffResponseTimeThreshold(String ns,
+ Configuration conf, int numLevels) {
+ long[] responseTimeThresholds = conf.getTimeDurations(ns + "." +
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY,
+ TimeUnit.MILLISECONDS);
+ // backoff thresholds not specified
+ if (responseTimeThresholds.length == 0) {
+ return getDefaultBackOffResponseTimeThresholds(numLevels);
+ }
+ // backoff thresholds specified but not match with the levels
+ if (responseTimeThresholds.length != numLevels) {
+ throw new IllegalArgumentException(
+ "responseTimeThresholds must match with the number of priority " +
+ "levels");
+ }
+ // invalid thresholds
+ for (long responseTimeThreshold: responseTimeThresholds) {
+ if (responseTimeThreshold <= 0) {
+ throw new IllegalArgumentException(
+ "responseTimeThreshold millis must be >= 0");
+ }
+ }
+ return responseTimeThresholds;
+ }
+
+ // 10s for level 0, 20s for level 1, 30s for level 2, ...
+ private static long[] getDefaultBackOffResponseTimeThresholds(int numLevels) {
+ long[] ret = new long[numLevels];
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = 10000*(i+1);
+ }
+ return ret;
+ }
+
+ private static Boolean parseBackOffByResponseTimeEnabled(String ns,
+ Configuration conf) {
+ return conf.getBoolean(ns + "." +
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY,
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_DEFAULT);
+ }
+
+ /**
+ * Decay the stored costs for each user and clean as necessary.
+ * This method should be called periodically in order to keep
+ * costs current.
+ */
+ private void decayCurrentCosts() {
+ LOG.debug("Start to decay current costs.");
+ try {
+ long totalDecayedCost = 0;
+ long totalRawCost = 0;
+ Iterator>> it =
+ callCosts.entrySet().iterator();
+
+ while (it.hasNext()) {
+ Map.Entry> entry = it.next();
+ AtomicLong decayedCost = entry.getValue().get(0);
+ AtomicLong rawCost = entry.getValue().get(1);
+
+
+ // Compute the next value by reducing it by the decayFactor
+ totalRawCost += rawCost.get();
+ long currentValue = decayedCost.get();
+ long nextValue = (long) (currentValue * decayFactor);
+ totalDecayedCost += nextValue;
+ decayedCost.set(nextValue);
+
+ LOG.debug(
+ "Decaying costs for the user: {}, its decayedCost: {}, rawCost: {}",
+ entry.getKey(), nextValue, rawCost.get());
+ if (nextValue == 0) {
+ LOG.debug("The decayed cost for the user {} is zero " +
+ "and being cleaned.", entry.getKey());
+ // We will clean up unused keys here. An interesting optimization
+ // might be to have an upper bound on keyspace in callCosts and only
+ // clean once we pass it.
+ it.remove();
+ }
+ }
+
+ // Update the total so that we remain in sync
+ totalDecayedCallCost.set(totalDecayedCost);
+ totalRawCallCost.set(totalRawCost);
+
+ LOG.debug("After decaying the stored costs, totalDecayedCost: {}, " +
+ "totalRawCallCost: {}.", totalDecayedCost, totalRawCost);
+ // Now refresh the cache of scheduling decisions
+ recomputeScheduleCache();
+
+ // Update average response time with decay
+ updateAverageResponseTime(true);
+ } catch (Exception ex) {
+ LOG.error("decayCurrentCosts exception: " +
+ ExceptionUtils.getStackTrace(ex));
+ throw ex;
+ }
+ }
+
+ /**
+ * Update the scheduleCache to match current conditions in callCosts.
+ */
+ private void recomputeScheduleCache() {
+ Map nextCache = new HashMap();
+
+ for (Map.Entry> entry : callCosts.entrySet()) {
+ Object id = entry.getKey();
+ AtomicLong value = entry.getValue().get(0);
+
+ long snapshot = value.get();
+ int computedLevel = computePriorityLevel(snapshot, id);
+
+ nextCache.put(id, computedLevel);
+ }
+
+ // Swap in to activate
+ scheduleCacheRef.set(Collections.unmodifiableMap(nextCache));
+ }
+
+ /**
+ * Adjust the stored cost for a given identity.
+ *
+ * @param identity the identity of the user whose cost should be adjusted
+ * @param costDelta the cost to add for the given identity
+ */
+ private void addCost(Object identity, long costDelta) {
+ // We will increment the cost, or create it if no such cost exists
+ List cost = this.callCosts.get(identity);
+ if (cost == null) {
+ // Create the costs since no such cost exists.
+ // idx 0 for decayed call cost
+ // idx 1 for the raw call cost
+ cost = new ArrayList(2);
+ cost.add(new AtomicLong(0));
+ cost.add(new AtomicLong(0));
+
+ // Put it in, or get the AtomicInteger that was put in by another thread
+ List otherCost = callCosts.putIfAbsent(identity, cost);
+ if (otherCost != null) {
+ cost = otherCost;
+ }
+ }
+
+ // Update the total
+ totalDecayedCallCost.getAndAdd(costDelta);
+ totalRawCallCost.getAndAdd(costDelta);
+
+ // At this point value is guaranteed to be not null. It may however have
+ // been clobbered from callCosts. Nonetheless, we return what
+ // we have.
+ cost.get(1).getAndAdd(costDelta);
+ cost.get(0).getAndAdd(costDelta);
+ }
+
+ /**
+ * Given the cost for an identity, compute a scheduling decision.
+ *
+ * @param cost the cost for an identity
+ * @return scheduling decision from 0 to numLevels - 1
+ */
+ private int computePriorityLevel(long cost, Object identity) {
+ Integer staticPriority = staticPriorities.get(identity);
+ if (staticPriority != null) {
+ return staticPriority.intValue();
+ }
+ long totalCallSnapshot = totalDecayedCallCost.get();
+
+ double proportion = 0;
+ if (totalCallSnapshot > 0) {
+ proportion = (double) cost / totalCallSnapshot;
+ }
+
+ // Start with low priority levels, since they will be most common
+ for(int i = (numLevels - 1); i > 0; i--) {
+ if (proportion >= this.thresholds[i - 1]) {
+ return i; // We've found our level number
+ }
+ }
+
+ // If we get this far, we're at level 0
+ return 0;
+ }
+
+ /**
+ * Returns the priority level for a given identity by first trying the cache,
+ * then computing it.
+ * @param identity an object responding to toString and hashCode
+ * @return integer scheduling decision from 0 to numLevels - 1
+ */
+ private int cachedOrComputedPriorityLevel(Object identity) {
+ // Try the cache
+ Map scheduleCache = scheduleCacheRef.get();
+ if (scheduleCache != null) {
+ Integer priority = scheduleCache.get(identity);
+ if (priority != null) {
+ LOG.debug("Cache priority for: {} with priority: {}", identity,
+ priority);
+ return priority;
+ }
+ }
+
+ // Cache was no good, compute it
+ List costList = callCosts.get(identity);
+ long currentCost = costList == null ? 0 : costList.get(0).get();
+ int priority = computePriorityLevel(currentCost, identity);
+ LOG.debug("compute priority for {} priority {}", identity, priority);
+ return priority;
+ }
+
+ private String getIdentity(Schedulable obj) {
+ String identity = this.identityProvider.makeIdentity(obj);
+ if (identity == null) {
+ // Identity provider did not handle this
+ identity = DECAYSCHEDULER_UNKNOWN_IDENTITY;
+ }
+ return identity;
+ }
+
+ /**
+ * Compute the appropriate priority for a schedulable based on past requests.
+ * @param obj the schedulable obj to query and remember
+ * @return the level index which we recommend scheduling in
+ */
+ @Override
+ public int getPriorityLevel(Schedulable obj) {
+ // First get the identity
+ String identity = getIdentity(obj);
+ // highest priority users may have a negative priority but their
+ // calls will be priority 0.
+ return Math.max(0, cachedOrComputedPriorityLevel(identity));
+ }
+
+ int getPriorityLevel(UserGroupInformation ugi) {
+ String identity = getIdentity(newSchedulable(ugi));
+ // returns true priority of the user.
+ return cachedOrComputedPriorityLevel(identity);
+ }
+
+ void setPriorityLevel(UserGroupInformation ugi, int priority) {
+ String identity = getIdentity(newSchedulable(ugi));
+ priority = Math.min(numLevels - 1, priority);
+ LOG.info("Setting priority for user:" + identity + "=" + priority);
+ staticPriorities.put(identity, priority);
+ }
+
+ // dummy instance to conform to identity provider api.
+ private static Schedulable newSchedulable(UserGroupInformation ugi) {
+ return new Schedulable() {
+ @Override
+ public UserGroupInformation getUserGroupInformation() {
+ return ugi;
+ }
+
+ @Override
+ public int getPriorityLevel() {
+ return 0;
+ }
+ };
+ }
+
+ @Override
+ public boolean shouldBackOff(Schedulable obj) {
+ Boolean backOff = false;
+ if (backOffByResponseTimeEnabled) {
+ int priorityLevel = obj.getPriorityLevel();
+ if (LOG.isDebugEnabled()) {
+ double[] responseTimes = getAverageResponseTime();
+ LOG.debug("Current Caller: {} Priority: {} ",
+ obj.getUserGroupInformation().getUserName(),
+ obj.getPriorityLevel());
+ for (int i = 0; i < numLevels; i++) {
+ LOG.debug("Queue: {} responseTime: {} backoffThreshold: {}", i,
+ responseTimes[i], backOffResponseTimeThresholds[i]);
+ }
+ }
+ // High priority rpc over threshold triggers back off of low priority rpc
+ for (int i = 0; i < priorityLevel + 1; i++) {
+ if (responseTimeAvgInLastWindow.get(i) >
+ backOffResponseTimeThresholds[i]) {
+ backOff = true;
+ break;
+ }
+ }
+ }
+ return backOff;
+ }
+
+ @Override
+ public void addResponseTime(String callName, Schedulable schedulable,
+ ProcessingDetails details) {
+ String user = identityProvider.makeIdentity(schedulable);
+ long processingCost = costProvider.getCost(details);
+ addCost(user, processingCost);
+
+ int priorityLevel = schedulable.getPriorityLevel();
+ long queueTime = details.get(Timing.QUEUE, RpcMetrics.TIMEUNIT);
+ long processingTime = details.get(Timing.PROCESSING, RpcMetrics.TIMEUNIT);
+
+ responseTimeCountInCurrWindow.getAndIncrement(priorityLevel);
+ responseTimeTotalInCurrWindow.getAndAdd(priorityLevel,
+ queueTime+processingTime);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("addResponseTime for call: {} priority: {} queueTime: {} " +
+ "processingTime: {} ", callName, priorityLevel, queueTime,
+ processingTime);
+ }
+ }
+
+ // Update the cached average response time at the end of the decay window
+ void updateAverageResponseTime(boolean enableDecay) {
+ for (int i = 0; i < numLevels; i++) {
+ double averageResponseTime = 0;
+ long totalResponseTime = responseTimeTotalInCurrWindow.get(i);
+ long responseTimeCount = responseTimeCountInCurrWindow.get(i);
+ if (responseTimeCount > 0) {
+ averageResponseTime = (double) totalResponseTime / responseTimeCount;
+ }
+ final double lastAvg = responseTimeAvgInLastWindow.get(i);
+ if (lastAvg > PRECISION || averageResponseTime > PRECISION) {
+ if (enableDecay) {
+ final double decayed = decayFactor * lastAvg + averageResponseTime;
+ responseTimeAvgInLastWindow.set(i, decayed);
+ } else {
+ responseTimeAvgInLastWindow.set(i, averageResponseTime);
+ }
+ } else {
+ responseTimeAvgInLastWindow.set(i, 0);
+ }
+ responseTimeCountInLastWindow.set(i, responseTimeCount);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updateAverageResponseTime queue: {} Average: {} Count: {}",
+ i, averageResponseTime, responseTimeCount);
+ }
+ // Reset for next decay window
+ responseTimeTotalInCurrWindow.set(i, 0);
+ responseTimeCountInCurrWindow.set(i, 0);
+ }
+ }
+
+ // For testing
+ double getDecayFactor() {
+ return decayFactor;
+ }
+
+ long getDecayPeriodMillis() {
+ return decayPeriodMillis;
+ }
+
+ double[] getThresholds() {
+ return thresholds;
+ }
+
+ void forceDecay() {
+ decayCurrentCosts();
+ }
+
+ Map getCallCostSnapshot() {
+ HashMap snapshot = new HashMap();
+
+ for (Map.Entry> entry : callCosts.entrySet()) {
+ snapshot.put(entry.getKey(), entry.getValue().get(0).get());
+ }
+
+ return Collections.unmodifiableMap(snapshot);
+ }
+
+ long getTotalCallSnapshot() {
+ return totalDecayedCallCost.get();
+ }
+
+ /**
+ * MetricsProxy is a singleton because we may init multiple schedulers and we
+ * want to clean up resources when a new scheduler replaces the old one.
+ */
+ public static final class MetricsProxy implements DecayRpcSchedulerMXBean,
+ MetricsSource {
+ // One singleton per namespace
+ private static final HashMap INSTANCES =
+ new HashMap();
+
+ // Weakref for delegate, so we don't retain it forever if it can be GC'd
+ private WeakReference delegate;
+ private double[] averageResponseTimeDefault;
+ private long[] callCountInLastWindowDefault;
+ private ObjectName decayRpcSchedulerInfoBeanName;
+
+ private MetricsProxy(String namespace, int numLevels,
+ DecayRpcScheduler drs) {
+ averageResponseTimeDefault = new double[numLevels];
+ callCountInLastWindowDefault = new long[numLevels];
+ setDelegate(drs);
+ decayRpcSchedulerInfoBeanName =
+ MBeans.register(namespace, "DecayRpcScheduler", this);
+ this.registerMetrics2Source(namespace);
+ }
+
+ public static synchronized MetricsProxy getInstance(String namespace,
+ int numLevels, DecayRpcScheduler drs) {
+ MetricsProxy mp = INSTANCES.get(namespace);
+ if (mp == null) {
+ // We must create one
+ mp = new MetricsProxy(namespace, numLevels, drs);
+ INSTANCES.put(namespace, mp);
+ } else if (drs != mp.delegate.get()){
+ // in case of delegate is reclaimed, we should set it again
+ mp.setDelegate(drs);
+ }
+ return mp;
+ }
+
+ public static synchronized void removeInstance(String namespace) {
+ MetricsProxy.INSTANCES.remove(namespace);
+ }
+
+ public void setDelegate(DecayRpcScheduler obj) {
+ this.delegate = new WeakReference(obj);
+ }
+
+ void registerMetrics2Source(String namespace) {
+ final String name = "DecayRpcSchedulerMetrics2." + namespace;
+ DefaultMetricsSystem.instance().register(name, name, this);
+ }
+
+ void unregisterSource(String namespace) {
+ final String name = "DecayRpcSchedulerMetrics2." + namespace;
+ DefaultMetricsSystem.instance().unregisterSource(name);
+ if (decayRpcSchedulerInfoBeanName != null) {
+ MBeans.unregister(decayRpcSchedulerInfoBeanName);
+ }
+ }
+
+ @Override
+ public String getSchedulingDecisionSummary() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return "No Active Scheduler";
+ } else {
+ return scheduler.getSchedulingDecisionSummary();
+ }
+ }
+
+ @Override
+ public String getCallVolumeSummary() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return "No Active Scheduler";
+ } else {
+ return scheduler.getCallVolumeSummary();
+ }
+ }
+
+ @Override
+ public int getUniqueIdentityCount() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return -1;
+ } else {
+ return scheduler.getUniqueIdentityCount();
+ }
+ }
+
+ @Override
+ public long getTotalCallVolume() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return -1;
+ } else {
+ return scheduler.getTotalCallVolume();
+ }
+ }
+
+ @Override
+ public double[] getAverageResponseTime() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return averageResponseTimeDefault;
+ } else {
+ return scheduler.getAverageResponseTime();
+ }
+ }
+
+ public long[] getResponseTimeCountInLastWindow() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return callCountInLastWindowDefault;
+ } else {
+ return scheduler.getResponseTimeCountInLastWindow();
+ }
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler != null) {
+ scheduler.getMetrics(collector, all);
+ }
+ }
+ }
+
+ public int getUniqueIdentityCount() {
+ return callCosts.size();
+ }
+
+ public long getTotalCallVolume() {
+ return totalDecayedCallCost.get();
+ }
+
+ public long getTotalRawCallVolume() {
+ return totalRawCallCost.get();
+ }
+
+ public long[] getResponseTimeCountInLastWindow() {
+ long[] ret = new long[responseTimeCountInLastWindow.length()];
+ for (int i = 0; i < responseTimeCountInLastWindow.length(); i++) {
+ ret[i] = responseTimeCountInLastWindow.get(i);
+ }
+ return ret;
+ }
+
+ @Override
+ public double[] getAverageResponseTime() {
+ double[] ret = new double[responseTimeAvgInLastWindow.length()];
+ for (int i = 0; i < responseTimeAvgInLastWindow.length(); i++) {
+ ret[i] = responseTimeAvgInLastWindow.get(i);
+ }
+ return ret;
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ // Metrics2 interface to act as a Metric source
+ try {
+ MetricsRecordBuilder rb = collector.addRecord(getClass().getName())
+ .setContext(namespace);
+ addDecayedCallVolume(rb);
+ addUniqueIdentityCount(rb);
+ addTopNCallerSummary(rb);
+ addAvgResponseTimePerPriority(rb);
+ addCallVolumePerPriority(rb);
+ addRawCallVolume(rb);
+ } catch (Exception e) {
+ LOG.warn("Exception thrown while metric collection. Exception : "
+ + e.getMessage());
+ }
+ }
+
+ // Key: UniqueCallers
+ private void addUniqueIdentityCount(MetricsRecordBuilder rb) {
+ rb.addCounter(Interns.info("UniqueCallers", "Total unique callers"),
+ getUniqueIdentityCount());
+ }
+
+ // Key: DecayedCallVolume
+ private void addDecayedCallVolume(MetricsRecordBuilder rb) {
+ rb.addCounter(Interns.info("DecayedCallVolume", "Decayed Total " +
+ "incoming Call Volume"), getTotalCallVolume());
+ }
+
+ private void addRawCallVolume(MetricsRecordBuilder rb) {
+ rb.addCounter(Interns.info("CallVolume", "Raw Total " +
+ "incoming Call Volume"), getTotalRawCallVolume());
+ }
+
+ // Key: Priority.0.CompletedCallVolume
+ private void addCallVolumePerPriority(MetricsRecordBuilder rb) {
+ for (int i = 0; i < responseTimeCountInLastWindow.length(); i++) {
+ rb.addGauge(Interns.info("Priority." + i + ".CompletedCallVolume",
+ "Completed Call volume " +
+ "of priority "+ i), responseTimeCountInLastWindow.get(i));
+ }
+ }
+
+ // Key: Priority.0.AvgResponseTime
+ private void addAvgResponseTimePerPriority(MetricsRecordBuilder rb) {
+ for (int i = 0; i < responseTimeAvgInLastWindow.length(); i++) {
+ rb.addGauge(Interns.info("Priority." + i + ".AvgResponseTime", "Average" +
+ " response time of priority " + i),
+ responseTimeAvgInLastWindow.get(i));
+ }
+ }
+
+ // Key: Caller(xyz).Volume and Caller(xyz).Priority
+ private void addTopNCallerSummary(MetricsRecordBuilder rb) {
+ TopN topNCallers = getTopCallers(topUsersCount);
+ Map decisions = scheduleCacheRef.get();
+ final int actualCallerCount = topNCallers.size();
+ for (int i = 0; i < actualCallerCount; i++) {
+ NameValuePair entry = topNCallers.poll();
+ String topCaller = "Caller(" + entry.getName() + ")";
+ String topCallerVolume = topCaller + ".Volume";
+ String topCallerPriority = topCaller + ".Priority";
+ rb.addCounter(Interns.info(topCallerVolume, topCallerVolume),
+ entry.getValue());
+ Integer priority = decisions.get(entry.getName());
+ if (priority != null) {
+ rb.addCounter(Interns.info(topCallerPriority, topCallerPriority),
+ priority);
+ }
+ }
+ }
+
+ // Get the top N callers' raw call cost and scheduler decision
+ private TopN getTopCallers(int n) {
+ TopN topNCallers = new TopN(n);
+ Iterator>> it =
+ callCosts.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry> entry = it.next();
+ String caller = entry.getKey().toString();
+ Long cost = entry.getValue().get(1).get();
+ if (cost > 0) {
+ topNCallers.offer(new NameValuePair(caller, cost));
+ }
+ }
+ return topNCallers;
+ }
+
+ public String getSchedulingDecisionSummary() {
+ Map decisions = scheduleCacheRef.get();
+ if (decisions == null) {
+ return "{}";
+ } else {
+ try {
+ return WRITER.writeValueAsString(decisions);
+ } catch (Exception e) {
+ return "Error: " + e.getMessage();
+ }
+ }
+ }
+
+ public String getCallVolumeSummary() {
+ try {
+ return WRITER.writeValueAsString(getDecayedCallCosts());
+ } catch (Exception e) {
+ return "Error: " + e.getMessage();
+ }
+ }
+
+ private Map getDecayedCallCosts() {
+ Map decayedCallCosts = new HashMap<>(callCosts.size());
+ Iterator>> it =
+ callCosts.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry> entry = it.next();
+ Object user = entry.getKey();
+ Long decayedCost = entry.getValue().get(0).get();
+ if (decayedCost > 0) {
+ decayedCallCosts.put(user, decayedCost);
+ }
+ }
+ return decayedCallCosts;
+ }
+
+ @Override
+ public void stop() {
+ metricsProxy.unregisterSource(namespace);
+ MetricsProxy.removeInstance(namespace);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DecayRpcSchedulerMXBean.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DecayRpcSchedulerMXBean.java
new file mode 100644
index 000000000000..a7ef13094d1a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DecayRpcSchedulerMXBean.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+/**
+ * Provides metrics for Decay scheduler.
+ */
+public interface DecayRpcSchedulerMXBean {
+ // Get an overview of the requests in history.
+ String getSchedulingDecisionSummary();
+ String getCallVolumeSummary();
+ int getUniqueIdentityCount();
+ long getTotalCallVolume();
+ double[] getAverageResponseTime();
+ long[] getResponseTimeCountInLastWindow();
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DefaultCostProvider.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DefaultCostProvider.java
new file mode 100644
index 000000000000..fd158e5824f0
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DefaultCostProvider.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Ignores process details and returns a constant value for each call.
+ */
+public class DefaultCostProvider implements CostProvider {
+
+ @Override
+ public void init(String namespace, Configuration conf) {
+ // No-op
+ }
+
+ /**
+ * Returns 1, regardless of the processing details.
+ *
+ * @param details Process details (ignored)
+ * @return 1
+ */
+ @Override
+ public long getCost(ProcessingDetails details) {
+ return 1;
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DefaultRpcScheduler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DefaultRpcScheduler.java
new file mode 100644
index 000000000000..a40850715a73
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/DefaultRpcScheduler.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * No op default RPC scheduler.
+ */
+public class DefaultRpcScheduler implements RpcScheduler {
+ @Override
+ public int getPriorityLevel(Schedulable obj) {
+ return 0;
+ }
+
+ @Override
+ public boolean shouldBackOff(Schedulable obj) {
+ return false;
+ }
+
+ @Override
+ public void addResponseTime(String callName, Schedulable schedulable,
+ ProcessingDetails details) {
+ }
+
+ public DefaultRpcScheduler(int priorityLevels, String namespace,
+ Configuration conf) {
+ }
+
+ @Override
+ public void stop() {
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ExternalCall.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ExternalCall.java
new file mode 100644
index 000000000000..a1618b990822
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ExternalCall.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.ipc_.Server.Call;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
+import org.apache.hadoop.security.UserGroupInformation;
+
+public abstract class ExternalCall extends Call {
+ private final PrivilegedExceptionAction action;
+ private final AtomicBoolean done = new AtomicBoolean();
+ private T result;
+ private Throwable error;
+
+ public ExternalCall(PrivilegedExceptionAction action) {
+ this.action = action;
+ }
+
+ @Override
+ public String getDetailedMetricsName() {
+ return "(external)";
+ }
+
+ public abstract UserGroupInformation getRemoteUser();
+
+ public final T get() throws InterruptedException, ExecutionException {
+ waitForCompletion();
+ if (error != null) {
+ throw new ExecutionException(error);
+ }
+ return result;
+ }
+
+ // wait for response to be triggered to support postponed calls
+ private void waitForCompletion() throws InterruptedException {
+ synchronized(done) {
+ while (!done.get()) {
+ try {
+ done.wait();
+ } catch (InterruptedException ie) {
+ if (Thread.interrupted()) {
+ throw ie;
+ }
+ }
+ }
+ }
+ }
+
+ boolean isDone() {
+ return done.get();
+ }
+
+ // invoked by ipc handler
+ @Override
+ public final Void run() throws IOException {
+ try {
+ result = action.run();
+ sendResponse();
+ } catch (Throwable t) {
+ abortResponse(t);
+ }
+ return null;
+ }
+
+ @Override
+ final void doResponse(Throwable t, RpcStatusProto status) {
+ synchronized(done) {
+ error = t;
+ done.set(true);
+ done.notify();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/FairCallQueue.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/FairCallQueue.java
new file mode 100644
index 000000000000..f03b1a9e75d4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/FairCallQueue.java
@@ -0,0 +1,453 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.lang.ref.WeakReference;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.AbstractQueue;
+import java.util.HashMap;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc_.CallQueueManager.CallQueueOverflowException;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A queue with multiple levels for each priority.
+ */
+public class FairCallQueue extends AbstractQueue
+ implements BlockingQueue {
+ @Deprecated
+ public static final int IPC_CALLQUEUE_PRIORITY_LEVELS_DEFAULT = 4;
+ @Deprecated
+ public static final String IPC_CALLQUEUE_PRIORITY_LEVELS_KEY =
+ "faircallqueue.priority-levels";
+
+ public static final Logger LOG = LoggerFactory.getLogger(FairCallQueue.class);
+
+ /* The queues */
+ private final ArrayList> queues;
+
+ /* Track available permits for scheduled objects. All methods that will
+ * mutate a subqueue must acquire or release a permit on the semaphore.
+ * A semaphore is much faster than an exclusive lock because producers do
+ * not contend with consumers and consumers do not block other consumers
+ * while polling.
+ */
+ private final Semaphore semaphore = new Semaphore(0);
+ private void signalNotEmpty() {
+ semaphore.release();
+ }
+
+ /* Multiplexer picks which queue to draw from */
+ private RpcMultiplexer multiplexer;
+
+ /* Statistic tracking */
+ private final ArrayList overflowedCalls;
+
+ /**
+ * Create a FairCallQueue.
+ * @param capacity the total size of all sub-queues
+ * @param ns the prefix to use for configuration
+ * @param conf the configuration to read from
+ * Notes: Each sub-queue has a capacity of `capacity / numSubqueues`.
+ * The first or the highest priority sub-queue has an excess capacity
+ * of `capacity % numSubqueues`
+ */
+ public FairCallQueue(int priorityLevels, int capacity, String ns,
+ Configuration conf) {
+ if(priorityLevels < 1) {
+ throw new IllegalArgumentException("Number of Priority Levels must be " +
+ "at least 1");
+ }
+ int numQueues = priorityLevels;
+ LOG.info("FairCallQueue is in use with " + numQueues +
+ " queues with total capacity of " + capacity);
+
+ this.queues = new ArrayList>(numQueues);
+ this.overflowedCalls = new ArrayList(numQueues);
+ int queueCapacity = capacity / numQueues;
+ int capacityForFirstQueue = queueCapacity + (capacity % numQueues);
+ for(int i=0; i < numQueues; i++) {
+ if (i == 0) {
+ this.queues.add(new LinkedBlockingQueue(capacityForFirstQueue));
+ } else {
+ this.queues.add(new LinkedBlockingQueue(queueCapacity));
+ }
+ this.overflowedCalls.add(new AtomicLong(0));
+ }
+
+ this.multiplexer = new WeightedRoundRobinMultiplexer(numQueues, ns, conf);
+ // Make this the active source of metrics
+ MetricsProxy mp = MetricsProxy.getInstance(ns);
+ mp.setDelegate(this);
+ }
+
+ /**
+ * Returns an element first non-empty queue equal to the priority returned
+ * by the multiplexer or scans from highest to lowest priority queue.
+ *
+ * Caller must always acquire a semaphore permit before invoking.
+ *
+ * @return the first non-empty queue with less priority, or null if
+ * everything was empty
+ */
+ private E removeNextElement() {
+ int priority = multiplexer.getAndAdvanceCurrentIndex();
+ E e = queues.get(priority).poll();
+ // a semaphore permit has been acquired, so an element MUST be extracted
+ // or the semaphore and queued elements will go out of sync. loop to
+ // avoid race condition if elements are added behind the current position,
+ // awakening other threads that poll the elements ahead of our position.
+ while (e == null) {
+ for (int idx = 0; e == null && idx < queues.size(); idx++) {
+ e = queues.get(idx).poll();
+ }
+ }
+ return e;
+ }
+
+ /* AbstractQueue and BlockingQueue methods */
+
+ /**
+ * Add, put, and offer follow the same pattern:
+ * 1. Get the assigned priorityLevel from the call by scheduler
+ * 2. Get the nth sub-queue matching this priorityLevel
+ * 3. delegate the call to this sub-queue.
+ *
+ * But differ in how they handle overflow:
+ * - Add will move on to the next queue, throw on last queue overflow
+ * - Put will move on to the next queue, block on last queue overflow
+ * - Offer does not attempt other queues on overflow
+ */
+
+ @Override
+ public boolean add(E e) {
+ final int priorityLevel = e.getPriorityLevel();
+ // try offering to all queues.
+ if (!offerQueues(priorityLevel, e, true)) {
+ // only disconnect the lowest priority users that overflow the queue.
+ throw (priorityLevel == queues.size() - 1)
+ ? CallQueueOverflowException.DISCONNECT
+ : CallQueueOverflowException.KEEPALIVE;
+ }
+ return true;
+ }
+
+ @Override
+ public void put(E e) throws InterruptedException {
+ final int priorityLevel = e.getPriorityLevel();
+ // try offering to all but last queue, put on last.
+ if (!offerQueues(priorityLevel, e, false)) {
+ putQueue(queues.size() - 1, e);
+ }
+ }
+
+ /**
+ * Put the element in a queue of a specific priority.
+ * @param priority - queue priority
+ * @param e - element to add
+ */
+ void putQueue(int priority, E e) throws InterruptedException {
+ queues.get(priority).put(e);
+ signalNotEmpty();
+ }
+
+ /**
+ * Offer the element to queue of a specific priority.
+ * @param priority - queue priority
+ * @param e - element to add
+ * @return boolean if added to the given queue
+ */
+ boolean offerQueue(int priority, E e) {
+ boolean ret = queues.get(priority).offer(e);
+ if (ret) {
+ signalNotEmpty();
+ }
+ return ret;
+ }
+
+ /**
+ * Offer the element to queue of the given or lower priority.
+ * @param priority - starting queue priority
+ * @param e - element to add
+ * @param includeLast - whether to attempt last queue
+ * @return boolean if added to a queue
+ */
+ private boolean offerQueues(int priority, E e, boolean includeLast) {
+ int lastPriority = queues.size() - (includeLast ? 1 : 2);
+ for (int i=priority; i <= lastPriority; i++) {
+ if (offerQueue(i, e)) {
+ return true;
+ }
+ // Update stats
+ overflowedCalls.get(i).getAndIncrement();
+ }
+ return false;
+ }
+
+ @Override
+ public boolean offer(E e, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ int priorityLevel = e.getPriorityLevel();
+ BlockingQueue q = this.queues.get(priorityLevel);
+ boolean ret = q.offer(e, timeout, unit);
+ if (ret) {
+ signalNotEmpty();
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean offer(E e) {
+ int priorityLevel = e.getPriorityLevel();
+ BlockingQueue q = this.queues.get(priorityLevel);
+ boolean ret = q.offer(e);
+ if (ret) {
+ signalNotEmpty();
+ }
+ return ret;
+ }
+
+ @Override
+ public E take() throws InterruptedException {
+ semaphore.acquire();
+ return removeNextElement();
+ }
+
+ @Override
+ public E poll(long timeout, TimeUnit unit) throws InterruptedException {
+ return semaphore.tryAcquire(timeout, unit) ? removeNextElement() : null;
+ }
+
+ /**
+ * poll() provides no strict consistency: it is possible for poll to return
+ * null even though an element is in the queue.
+ */
+ @Override
+ public E poll() {
+ return semaphore.tryAcquire() ? removeNextElement() : null;
+ }
+
+ /**
+ * Peek, like poll, provides no strict consistency.
+ */
+ @Override
+ public E peek() {
+ E e = null;
+ for (int i=0; e == null && i < queues.size(); i++) {
+ e = queues.get(i).peek();
+ }
+ return e;
+ }
+
+ /**
+ * Size returns the sum of all sub-queue sizes, so it may be greater than
+ * capacity.
+ * Note: size provides no strict consistency, and should not be used to
+ * control queue IO.
+ */
+ @Override
+ public int size() {
+ return semaphore.availablePermits();
+ }
+
+ /**
+ * Iterator is not implemented, as it is not needed.
+ */
+ @Override
+ public Iterator iterator() {
+ throw new NotImplementedException("Code is not implemented");
+ }
+
+ /**
+ * drainTo defers to each sub-queue. Note that draining from a FairCallQueue
+ * to another FairCallQueue will likely fail, since the incoming calls
+ * may be scheduled differently in the new FairCallQueue. Nonetheless this
+ * method is provided for completeness.
+ */
+ @Override
+ public int drainTo(Collection c, int maxElements) {
+ // initially take all permits to stop consumers from modifying queues
+ // while draining. will restore any excess when done draining.
+ final int permits = semaphore.drainPermits();
+ final int numElements = Math.min(maxElements, permits);
+ int numRemaining = numElements;
+ for (int i=0; numRemaining > 0 && i < queues.size(); i++) {
+ numRemaining -= queues.get(i).drainTo(c, numRemaining);
+ }
+ int drained = numElements - numRemaining;
+ if (permits > drained) { // restore unused permits.
+ semaphore.release(permits - drained);
+ }
+ return drained;
+ }
+
+ @Override
+ public int drainTo(Collection c) {
+ return drainTo(c, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Returns maximum remaining capacity. This does not reflect how much you can
+ * ideally fit in this FairCallQueue, as that would depend on the scheduler's
+ * decisions.
+ */
+ @Override
+ public int remainingCapacity() {
+ int sum = 0;
+ for (BlockingQueue q : this.queues) {
+ sum += q.remainingCapacity();
+ }
+ return sum;
+ }
+
+ /**
+ * MetricsProxy is a singleton because we may init multiple
+ * FairCallQueues, but the metrics system cannot unregister beans cleanly.
+ */
+ private static final class MetricsProxy implements FairCallQueueMXBean,
+ MetricsSource {
+ // One singleton per namespace
+ private static final HashMap INSTANCES =
+ new HashMap();
+
+ // Weakref for delegate, so we don't retain it forever if it can be GC'd
+ private WeakReference> delegate;
+
+ // Keep track of how many objects we registered
+ private int revisionNumber = 0;
+
+ private String namespace;
+
+ private MetricsProxy(String namespace) {
+ this.namespace = namespace;
+ MBeans.register(namespace, "FairCallQueue", this);
+ final String name = namespace + ".FairCallQueue";
+ DefaultMetricsSystem.instance().register(name, name, this);
+ }
+
+ public static synchronized MetricsProxy getInstance(String namespace) {
+ MetricsProxy mp = INSTANCES.get(namespace);
+ if (mp == null) {
+ // We must create one
+ mp = new MetricsProxy(namespace);
+ INSTANCES.put(namespace, mp);
+ }
+ return mp;
+ }
+
+ public void setDelegate(FairCallQueue obj) {
+ this.delegate
+ = new WeakReference>(obj);
+ this.revisionNumber++;
+ }
+
+ /**
+ * Fetch the current call queue from the weak reference delegate. If there
+ * is no delegate, or the delegate is empty, this will return null.
+ */
+ private FairCallQueue getCallQueue() {
+ WeakReference> ref = this.delegate;
+ if (ref == null) {
+ return null;
+ }
+ return ref.get();
+ }
+
+ @Override
+ public int[] getQueueSizes() {
+ FairCallQueue obj = getCallQueue();
+ if (obj == null) {
+ return new int[]{};
+ }
+
+ return obj.getQueueSizes();
+ }
+
+ @Override
+ public long[] getOverflowedCalls() {
+ FairCallQueue obj = getCallQueue();
+ if (obj == null) {
+ return new long[]{};
+ }
+
+ return obj.getOverflowedCalls();
+ }
+
+ @Override public int getRevision() {
+ return revisionNumber;
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ MetricsRecordBuilder rb = collector.addRecord("FairCallQueue")
+ .setContext("rpc")
+ .tag(Interns.info("namespace", "Namespace"), namespace);
+
+ final int[] currentQueueSizes = getQueueSizes();
+ final long[] currentOverflowedCalls = getOverflowedCalls();
+
+ for (int i = 0; i < currentQueueSizes.length; i++) {
+ rb.addGauge(Interns.info("FairCallQueueSize_p" + i, "FCQ Queue Size"),
+ currentQueueSizes[i]);
+ rb.addCounter(Interns.info("FairCallQueueOverflowedCalls_p" + i,
+ "FCQ Overflowed Calls"), currentOverflowedCalls[i]);
+ }
+ }
+ }
+
+ // FairCallQueueMXBean
+ public int[] getQueueSizes() {
+ int numQueues = queues.size();
+ int[] sizes = new int[numQueues];
+ for (int i=0; i < numQueues; i++) {
+ sizes[i] = queues.get(i).size();
+ }
+ return sizes;
+ }
+
+ public long[] getOverflowedCalls() {
+ int numQueues = queues.size();
+ long[] calls = new long[numQueues];
+ for (int i=0; i < numQueues; i++) {
+ calls[i] = overflowedCalls.get(i).get();
+ }
+ return calls;
+ }
+
+ public void setMultiplexer(RpcMultiplexer newMux) {
+ this.multiplexer = newMux;
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/FairCallQueueMXBean.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/FairCallQueueMXBean.java
new file mode 100644
index 000000000000..f23e58cd8cb4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/FairCallQueueMXBean.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+public interface FairCallQueueMXBean {
+ // Get the size of each subqueue, the index corresponding to the priority
+ // level.
+ int[] getQueueSizes();
+ long[] getOverflowedCalls();
+ int getRevision();
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/GenericRefreshProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/GenericRefreshProtocol.java
new file mode 100644
index 000000000000..68027aa7ef4c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/GenericRefreshProtocol.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * Protocol which is used to refresh arbitrary things at runtime.
+ */
+@KerberosInfo(
+ serverPrincipal=CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY)
+public interface GenericRefreshProtocol {
+ /**
+ * Version 1: Initial version.
+ */
+ public static final long versionID = 1L;
+
+ /**
+ * Refresh the resource based on identity passed in.
+ *
+ * @param identifier input identifier.
+ * @param args input args.
+ * @throws IOException raised on errors performing I/O.
+ * @return Collection RefreshResponse.
+ */
+ @Idempotent
+ Collection refresh(String identifier, String[] args)
+ throws IOException;
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/IdentityProvider.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/IdentityProvider.java
new file mode 100644
index 000000000000..393e332d49e2
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/IdentityProvider.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+
+/**
+ * The IdentityProvider creates identities for each schedulable
+ * by extracting fields and returning an identity string.
+ *
+ * Implementers will be able to change how schedulers treat
+ * Schedulables.
+ */
+public interface IdentityProvider {
+ /**
+ * Return the string used for scheduling.
+ * @param obj the schedulable to use.
+ * @return string identity, or null if no identity could be made.
+ */
+ public String makeIdentity(Schedulable obj);
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/IpcException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/IpcException.java
new file mode 100644
index 000000000000..67257c45a991
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/IpcException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+
+/**
+ * IPC exception is thrown by IPC layer when the IPC
+ * connection cannot be established.
+ */
+public class IpcException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public IpcException(final String err) {
+ super(err);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ObserverRetryOnActiveException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ObserverRetryOnActiveException.java
new file mode 100644
index 000000000000..b32791bb1494
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ObserverRetryOnActiveException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+
+/**
+ * Thrown by a remote ObserverNode indicating the operation has failed and the
+ * client should retry active namenode directly (instead of retry other
+ * ObserverNodes).
+ */
+public class ObserverRetryOnActiveException extends StandbyException {
+ static final long serialVersionUID = 1L;
+ public ObserverRetryOnActiveException(String msg) {
+ super(msg);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProcessingDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProcessingDetails.java
new file mode 100644
index 000000000000..3e0eb79a08c0
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProcessingDetails.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Stores the times that a call takes to be processed through each step.
+ */
+public class ProcessingDetails {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ProcessingDetails.class);
+ private final TimeUnit valueTimeUnit;
+
+ /**
+ * The different stages to track the time of.
+ */
+ public enum Timing {
+ ENQUEUE, // time for reader to insert in call queue.
+ QUEUE, // time in the call queue.
+ HANDLER, // handler overhead not spent in processing/response.
+ PROCESSING, // time handler spent processing the call. always equal to
+ // lock_free + lock_wait + lock_shared + lock_exclusive
+ LOCKFREE, // processing with no lock.
+ LOCKWAIT, // processing while waiting for lock.
+ LOCKSHARED, // processing with a read lock.
+ LOCKEXCLUSIVE, // processing with a write lock.
+ RESPONSE; // time to encode and send response.
+ }
+
+ private long[] timings = new long[Timing.values().length];
+
+ ProcessingDetails(TimeUnit timeUnit) {
+ this.valueTimeUnit = timeUnit;
+ }
+
+ public long get(Timing type) {
+ // When using nanoTime to fetch timing information, it is possible to see
+ // time "move backward" slightly under unusual/rare circumstances. To avoid
+ // displaying a confusing number, round such timings to 0 here.
+ long ret = timings[type.ordinal()];
+ return ret < 0 ? 0 : ret;
+ }
+
+ public long get(Timing type, TimeUnit timeUnit) {
+ return timeUnit.convert(get(type), valueTimeUnit);
+ }
+
+ public void set(Timing type, long value) {
+ timings[type.ordinal()] = value;
+ }
+
+ public void set(Timing type, long value, TimeUnit timeUnit) {
+ set(type, valueTimeUnit.convert(value, timeUnit));
+ }
+
+ public void add(Timing type, long value, TimeUnit timeUnit) {
+ timings[type.ordinal()] += valueTimeUnit.convert(value, timeUnit);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ for (Timing type : Timing.values()) {
+ if (sb.length() > 0) {
+ sb.append(" ");
+ }
+ sb.append(type.name().toLowerCase())
+ .append("Time=").append(get(type));
+ }
+ return sb.toString();
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtoUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtoUtil.java
new file mode 100644
index 000000000000..2fe400b72174
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtoUtil.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.DataInput;
+import java.io.IOException;
+
+import org.apache.hadoop.ipc_.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
+import org.apache.hadoop.ipc_.protobuf.IpcConnectionContextProtos.UserInformationProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.*;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import com.google.protobuf.ByteString;
+
+public abstract class ProtoUtil {
+
+ /**
+ * Read a variable length integer in the same format that ProtoBufs encodes.
+ * @param in the input stream to read from
+ * @return the integer
+ * @throws IOException if it is malformed or EOF.
+ */
+ public static int readRawVarint32(DataInput in) throws IOException {
+ byte tmp = in.readByte();
+ if (tmp >= 0) {
+ return tmp;
+ }
+ int result = tmp & 0x7f;
+ if ((tmp = in.readByte()) >= 0) {
+ result |= tmp << 7;
+ } else {
+ result |= (tmp & 0x7f) << 7;
+ if ((tmp = in.readByte()) >= 0) {
+ result |= tmp << 14;
+ } else {
+ result |= (tmp & 0x7f) << 14;
+ if ((tmp = in.readByte()) >= 0) {
+ result |= tmp << 21;
+ } else {
+ result |= (tmp & 0x7f) << 21;
+ result |= (tmp = in.readByte()) << 28;
+ if (tmp < 0) {
+ // Discard upper 32 bits.
+ for (int i = 0; i < 5; i++) {
+ if (in.readByte() >= 0) {
+ return result;
+ }
+ }
+ throw new IOException("Malformed varint");
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+
+ /**
+ * This method creates the connection context using exactly the same logic
+ * as the old connection context as was done for writable where
+ * the effective and real users are set based on the auth method.
+ *
+ */
+ public static IpcConnectionContextProto makeIpcConnectionContext(
+ final String protocol,
+ final UserGroupInformation ugi, final AuthMethod authMethod) {
+ IpcConnectionContextProto.Builder result = IpcConnectionContextProto.newBuilder();
+ if (protocol != null) {
+ result.setProtocol(protocol);
+ }
+ UserInformationProto.Builder ugiProto = UserInformationProto.newBuilder();
+ if (ugi != null) {
+ /*
+ * In the connection context we send only additional user info that
+ * is not derived from the authentication done during connection setup.
+ */
+ if (authMethod == AuthMethod.KERBEROS) {
+ // Real user was established as part of the connection.
+ // Send effective user only.
+ ugiProto.setEffectiveUser(ugi.getUserName());
+ } else if (authMethod == AuthMethod.TOKEN) {
+ // With token, the connection itself establishes
+ // both real and effective user. Hence send none in header.
+ } else { // Simple authentication
+ // No user info is established as part of the connection.
+ // Send both effective user and real user
+ ugiProto.setEffectiveUser(ugi.getUserName());
+ if (ugi.getRealUser() != null) {
+ ugiProto.setRealUser(ugi.getRealUser().getUserName());
+ }
+ }
+ }
+ result.setUserInfo(ugiProto);
+ return result.build();
+ }
+
+ public static UserGroupInformation getUgi(IpcConnectionContextProto context) {
+ if (context.hasUserInfo()) {
+ UserInformationProto userInfo = context.getUserInfo();
+ return getUgi(userInfo);
+ } else {
+ return null;
+ }
+ }
+
+ public static UserGroupInformation getUgi(UserInformationProto userInfo) {
+ UserGroupInformation ugi = null;
+ String effectiveUser = userInfo.hasEffectiveUser() ? userInfo
+ .getEffectiveUser() : null;
+ String realUser = userInfo.hasRealUser() ? userInfo.getRealUser() : null;
+ if (effectiveUser != null) {
+ if (realUser != null) {
+ UserGroupInformation realUserUgi = UserGroupInformation
+ .createRemoteUser(realUser);
+ ugi = UserGroupInformation
+ .createProxyUser(effectiveUser, realUserUgi);
+ } else {
+ ugi = org.apache.hadoop.security.UserGroupInformation
+ .createRemoteUser(effectiveUser);
+ }
+ }
+ return ugi;
+ }
+
+ static RpcKindProto convert(RPC.RpcKind kind) {
+ switch (kind) {
+ case RPC_BUILTIN: return RpcKindProto.RPC_BUILTIN;
+ case RPC_WRITABLE: return RpcKindProto.RPC_WRITABLE;
+ case RPC_PROTOCOL_BUFFER: return RpcKindProto.RPC_PROTOCOL_BUFFER;
+ }
+ return null;
+ }
+
+
+ public static RPC.RpcKind convert( RpcKindProto kind) {
+ switch (kind) {
+ case RPC_BUILTIN: return RPC.RpcKind.RPC_BUILTIN;
+ case RPC_WRITABLE: return RPC.RpcKind.RPC_WRITABLE;
+ case RPC_PROTOCOL_BUFFER: return RPC.RpcKind.RPC_PROTOCOL_BUFFER;
+ }
+ return null;
+ }
+
+ public static RpcRequestHeaderProto makeRpcRequestHeader(RPC.RpcKind rpcKind,
+ RpcRequestHeaderProto.OperationProto operation, int callId,
+ int retryCount, byte[] uuid) {
+ return makeRpcRequestHeader(rpcKind, operation, callId, retryCount, uuid,
+ null);
+ }
+
+ public static RpcRequestHeaderProto makeRpcRequestHeader(RPC.RpcKind rpcKind,
+ RpcRequestHeaderProto.OperationProto operation, int callId,
+ int retryCount, byte[] uuid, AlignmentContext alignmentContext) {
+ RpcRequestHeaderProto.Builder result = RpcRequestHeaderProto.newBuilder();
+ result.setRpcKind(convert(rpcKind)).setRpcOp(operation).setCallId(callId)
+ .setRetryCount(retryCount).setClientId(ByteString.copyFrom(uuid));
+
+ // Add caller context if it is not null
+ CallerContext callerContext = CallerContext.getCurrent();
+ if (callerContext != null && callerContext.isContextValid()) {
+ RPCCallerContextProto.Builder contextBuilder = RPCCallerContextProto
+ .newBuilder().setContext(callerContext.getContext());
+ if (callerContext.getSignature() != null) {
+ contextBuilder.setSignature(
+ ByteString.copyFrom(callerContext.getSignature()));
+ }
+ result.setCallerContext(contextBuilder);
+ }
+
+ // Add alignment context if it is not null
+ if (alignmentContext != null) {
+ alignmentContext.updateRequestState(result);
+ }
+
+ return result.build();
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufHelper.java
new file mode 100644
index 000000000000..974442dc88fc
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufHelper.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+
+
+import com.google.protobuf.ServiceException;
+
+/**
+ * Helper methods for protobuf related RPC implementation
+ */
+public class ProtobufHelper {
+ private ProtobufHelper() {
+ // Hidden constructor for class with only static helper methods
+ }
+
+ /**
+ * Return the IOException thrown by the remote server wrapped in
+ * ServiceException as cause.
+ * @param se ServiceException that wraps IO exception thrown by the server
+ * @return Exception wrapped in ServiceException or
+ * a new IOException that wraps the unexpected ServiceException.
+ */
+ public static IOException getRemoteException(ServiceException se) {
+ Throwable e = se.getCause();
+ if (e == null) {
+ return new IOException(se);
+ }
+ return e instanceof IOException ? (IOException) e : new IOException(se);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufRpcEngine.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufRpcEngine.java
new file mode 100644
index 000000000000..d7d2d88259c7
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufRpcEngine.java
@@ -0,0 +1,605 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import com.google.protobuf.*;
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc_.Client.ConnectionId;
+import org.apache.hadoop.ipc_.RPC.RpcInvoker;
+import org.apache.hadoop.ipc_.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.AsyncGet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * RPC Engine for for protobuf based RPCs.
+ */
+public class ProtobufRpcEngine implements RpcEngine {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ProtobufRpcEngine.class);
+ private static final ThreadLocal>
+ ASYNC_RETURN_MESSAGE = new ThreadLocal<>();
+
+ static { // Register the rpcRequest deserializer for ProtobufRpcEngine
+ org.apache.hadoop.ipc_.Server.registerProtocolEngine(
+ RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcProtobufRequest.class,
+ new Server.ProtoBufRpcInvoker());
+ }
+
+ private static final ClientCache CLIENTS = new ClientCache();
+
+ public static AsyncGet getAsyncReturnMessage() {
+ return ASYNC_RETURN_MESSAGE.get();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public ProtocolProxy getProxy(Class protocol, long clientVersion,
+ ConnectionId connId, Configuration conf, SocketFactory factory)
+ throws IOException {
+ final Invoker invoker = new Invoker(protocol, connId, conf, factory);
+ return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance(
+ protocol.getClassLoader(), new Class[] {protocol}, invoker), false);
+ }
+
+ public ProtocolProxy getProxy(Class protocol, long clientVersion,
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout) throws IOException {
+ return getProxy(protocol, clientVersion, addr, ticket, conf, factory,
+ rpcTimeout, null);
+ }
+
+ @Override
+ public ProtocolProxy getProxy(Class protocol, long clientVersion,
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy
+ ) throws IOException {
+ return getProxy(protocol, clientVersion, addr, ticket, conf, factory,
+ rpcTimeout, connectionRetryPolicy, null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public ProtocolProxy getProxy(Class protocol, long clientVersion,
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth, AlignmentContext alignmentContext)
+ throws IOException {
+
+ final Invoker invoker = new Invoker(protocol, addr, ticket, conf, factory,
+ rpcTimeout, connectionRetryPolicy, fallbackToSimpleAuth,
+ alignmentContext);
+ return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance(
+ protocol.getClassLoader(), new Class[]{protocol}, invoker), false);
+ }
+
+ @Override
+ public ProtocolProxy getProtocolMetaInfoProxy(
+ ConnectionId connId, Configuration conf, SocketFactory factory)
+ throws IOException {
+ Class protocol = ProtocolMetaInfoPB.class;
+ return new ProtocolProxy(protocol,
+ (ProtocolMetaInfoPB) Proxy.newProxyInstance(protocol.getClassLoader(),
+ new Class[] { protocol }, new Invoker(protocol, connId, conf,
+ factory)), false);
+ }
+
+ protected static class Invoker implements RpcInvocationHandler {
+ private final Map returnTypes =
+ new ConcurrentHashMap();
+ private boolean isClosed = false;
+ private final Client.ConnectionId remoteId;
+ private final Client client;
+ private final long clientProtocolVersion;
+ private final String protocolName;
+ private AtomicBoolean fallbackToSimpleAuth;
+ private AlignmentContext alignmentContext;
+
+ protected Invoker(Class protocol, InetSocketAddress addr,
+ UserGroupInformation ticket, Configuration conf, SocketFactory factory,
+ int rpcTimeout, RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth, AlignmentContext alignmentContext)
+ throws IOException {
+ this(protocol, Client.ConnectionId.getConnectionId(
+ addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf),
+ conf, factory);
+ this.fallbackToSimpleAuth = fallbackToSimpleAuth;
+ this.alignmentContext = alignmentContext;
+ }
+
+ /**
+ * This constructor takes a connectionId, instead of creating a new one.
+ * @param protocol input protocol.
+ * @param connId input connId.
+ * @param conf input Configuration.
+ * @param factory input factory.
+ */
+ protected Invoker(Class protocol, Client.ConnectionId connId,
+ Configuration conf, SocketFactory factory) {
+ this.remoteId = connId;
+ this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class);
+ this.protocolName = RPC.getProtocolName(protocol);
+ this.clientProtocolVersion = RPC
+ .getProtocolVersion(protocol);
+ }
+
+ private RequestHeaderProto constructRpcRequestHeader(Method method) {
+ RequestHeaderProto.Builder builder = RequestHeaderProto
+ .newBuilder();
+ builder.setMethodName(method.getName());
+
+
+ // For protobuf, {@code protocol} used when creating client side proxy is
+ // the interface extending BlockingInterface, which has the annotations
+ // such as ProtocolName etc.
+ //
+ // Using Method.getDeclaringClass(), as in WritableEngine to get at
+ // the protocol interface will return BlockingInterface, from where
+ // the annotation ProtocolName and Version cannot be
+ // obtained.
+ //
+ // Hence we simply use the protocol class used to create the proxy.
+ // For PB this may limit the use of mixins on client side.
+ builder.setDeclaringClassProtocolName(protocolName);
+ builder.setClientProtocolVersion(clientProtocolVersion);
+ return builder.build();
+ }
+
+ /**
+ * This is the client side invoker of RPC method. It only throws
+ * ServiceException, since the invocation proxy expects only
+ * ServiceException to be thrown by the method in case protobuf service.
+ *
+ * ServiceException has the following causes:
+ *
+ * Exceptions encountered on the client side in this method are
+ * set as cause in ServiceException as is.
+ * Exceptions from the server are wrapped in RemoteException and are
+ * set as cause in ServiceException
+ *
+ *
+ * Note that the client calling protobuf RPC methods, must handle
+ * ServiceException by getting the cause from the ServiceException. If the
+ * cause is RemoteException, then unwrap it to get the exception thrown by
+ * the server.
+ */
+ @Override
+ public Message invoke(Object proxy, final Method method, Object[] args)
+ throws ServiceException {
+ long startTime = 0;
+ if (LOG.isDebugEnabled()) {
+ startTime = Time.now();
+ }
+
+ if (args.length != 2) { // RpcController + Message
+ throw new ServiceException(
+ "Too many or few parameters for request. Method: ["
+ + method.getName() + "]" + ", Expected: 2, Actual: "
+ + args.length);
+ }
+ if (args[1] == null) {
+ throw new ServiceException("null param while calling Method: ["
+ + method.getName() + "]");
+ }
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(Thread.currentThread().getId() + ": Call -> " +
+ remoteId + ": " + method.getName() +
+ " {" + TextFormat.shortDebugString((Message) args[1]) + "}");
+ }
+
+
+ final Message theRequest = (Message) args[1];
+ final RpcWritable.Buffer val;
+ try {
+ val = (RpcWritable.Buffer) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ constructRpcRequest(method, theRequest), remoteId,
+ fallbackToSimpleAuth, alignmentContext);
+
+ } catch (Throwable e) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(Thread.currentThread().getId() + ": Exception <- " +
+ remoteId + ": " + method.getName() +
+ " {" + e + "}");
+ }
+ throw new ServiceException(e);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ long callTime = Time.now() - startTime;
+ LOG.debug("Call: " + method.getName() + " took " + callTime + "ms");
+ }
+
+ if (Client.isAsynchronousMode()) {
+ final AsyncGet arr
+ = Client.getAsyncRpcResponse();
+ final AsyncGet asyncGet
+ = new AsyncGet() {
+ @Override
+ public Message get(long timeout, TimeUnit unit) throws Exception {
+ return getReturnMessage(method, arr.get(timeout, unit));
+ }
+
+ @Override
+ public boolean isDone() {
+ return arr.isDone();
+ }
+ };
+ ASYNC_RETURN_MESSAGE.set(asyncGet);
+ return null;
+ } else {
+ return getReturnMessage(method, val);
+ }
+ }
+
+ protected Writable constructRpcRequest(Method method, Message theRequest) {
+ RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method);
+ return new RpcProtobufRequest(rpcRequestHeader, theRequest);
+ }
+
+ private Message getReturnMessage(final Method method,
+ final RpcWritable.Buffer buf) throws ServiceException {
+ Message prototype = null;
+ try {
+ prototype = getReturnProtoType(method);
+ } catch (Exception e) {
+ throw new ServiceException(e);
+ }
+ Message returnMessage;
+ try {
+ returnMessage = buf.getValue(prototype.getDefaultInstanceForType());
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(Thread.currentThread().getId() + ": Response <- " +
+ remoteId + ": " + method.getName() +
+ " {" + TextFormat.shortDebugString(returnMessage) + "}");
+ }
+
+ } catch (Throwable e) {
+ throw new ServiceException(e);
+ }
+ return returnMessage;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!isClosed) {
+ isClosed = true;
+ CLIENTS.stopClient(client);
+ }
+ }
+
+ private Message getReturnProtoType(Method method) throws Exception {
+ if (returnTypes.containsKey(method.getName())) {
+ return returnTypes.get(method.getName());
+ }
+
+ Class returnType = method.getReturnType();
+ Method newInstMethod = returnType.getMethod("getDefaultInstance");
+ newInstMethod.setAccessible(true);
+ Message prototype = (Message) newInstMethod.invoke(null, (Object[]) null);
+ returnTypes.put(method.getName(), prototype);
+ return prototype;
+ }
+
+ @Override //RpcInvocationHandler
+ public ConnectionId getConnectionId() {
+ return remoteId;
+ }
+
+ protected long getClientProtocolVersion() {
+ return clientProtocolVersion;
+ }
+
+ protected String getProtocolName() {
+ return protocolName;
+ }
+ }
+
+ static Client getClient(Configuration conf) {
+ return CLIENTS.getClient(conf, SocketFactory.getDefault(),
+ RpcWritable.Buffer.class);
+ }
+
+ public static void clearClientCache() {
+ CLIENTS.clearCache();
+ }
+
+ @Override
+ public RPC.Server getServer(Class protocol, Object protocolImpl,
+ String bindAddress, int port, int numHandlers, int numReaders,
+ int queueSizePerHandler, boolean verbose, Configuration conf,
+ SecretManager secretManager,
+ String portRangeConfig, AlignmentContext alignmentContext)
+ throws IOException {
+ return new Server(protocol, protocolImpl, conf, bindAddress, port,
+ numHandlers, numReaders, queueSizePerHandler, verbose, secretManager,
+ portRangeConfig, alignmentContext);
+ }
+
+ public static class Server extends RPC.Server {
+
+ static final ThreadLocal currentCallback =
+ new ThreadLocal<>();
+
+ static final ThreadLocal currentCallInfo = new ThreadLocal<>();
+
+ static class CallInfo {
+ private final RPC.Server server;
+ private final String methodName;
+
+ public CallInfo(RPC.Server server, String methodName) {
+ this.server = server;
+ this.methodName = methodName;
+ }
+ }
+
+ static class ProtobufRpcEngineCallbackImpl
+ implements ProtobufRpcEngineCallback {
+
+ private final RPC.Server server;
+ private final Call call;
+ private final String methodName;
+ private final long setupTime;
+
+ public ProtobufRpcEngineCallbackImpl() {
+ this.server = currentCallInfo.get().server;
+ this.call = Server.getCurCall().get();
+ this.methodName = currentCallInfo.get().methodName;
+ this.setupTime = Time.now();
+ }
+
+ @Override
+ public void setResponse(Message message) {
+ long processingTime = Time.now() - setupTime;
+ call.setDeferredResponse(RpcWritable.wrap(message));
+ server.updateDeferredMetrics(methodName, processingTime);
+ }
+
+ @Override
+ public void error(Throwable t) {
+ long processingTime = Time.now() - setupTime;
+ String detailedMetricsName = t.getClass().getSimpleName();
+ server.updateDeferredMetrics(detailedMetricsName, processingTime);
+ call.setDeferredError(t);
+ }
+ }
+
+ public static ProtobufRpcEngineCallback registerForDeferredResponse() {
+ ProtobufRpcEngineCallback callback = new ProtobufRpcEngineCallbackImpl();
+ currentCallback.set(callback);
+ return callback;
+ }
+
+ /**
+ * Construct an RPC server.
+ *
+ * @param protocolClass the class of protocol
+ * @param protocolImpl the protocolImpl whose methods will be called
+ * @param conf the configuration to use
+ * @param bindAddress the address to bind on to listen for connection
+ * @param port the port to listen for connections on
+ * @param numHandlers the number of method handler threads to run
+ * @param verbose whether each call should be logged
+ * @param portRangeConfig A config parameter that can be used to restrict
+ * @param alignmentContext provides server state info on client responses
+ * @param secretManager input secretManager.
+ * @param queueSizePerHandler input queueSizePerHandler.
+ * @param numReaders input numReaders.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public Server(Class protocolClass, Object protocolImpl,
+ Configuration conf, String bindAddress, int port, int numHandlers,
+ int numReaders, int queueSizePerHandler, boolean verbose,
+ SecretManager secretManager,
+ String portRangeConfig, AlignmentContext alignmentContext)
+ throws IOException {
+ super(bindAddress, port, null, numHandlers,
+ numReaders, queueSizePerHandler, conf,
+ serverNameFromClass(protocolImpl.getClass()), secretManager,
+ portRangeConfig);
+ setAlignmentContext(alignmentContext);
+ this.verbose = verbose;
+ registerProtocolAndImpl(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocolClass,
+ protocolImpl);
+ }
+
+ /**
+ * Protobuf invoker for {@link RpcInvoker}
+ */
+ static class ProtoBufRpcInvoker implements RpcInvoker {
+ private static ProtoClassProtoImpl getProtocolImpl(RPC.Server server,
+ String protoName, long clientVersion) throws RpcServerException {
+ ProtoNameVer pv = new ProtoNameVer(protoName, clientVersion);
+ ProtoClassProtoImpl impl =
+ server.getProtocolImplMap(RPC.RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
+ if (impl == null) { // no match for Protocol AND Version
+ VerProtocolImpl highest =
+ server.getHighestSupportedProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ protoName);
+ if (highest == null) {
+ throw new RpcNoSuchProtocolException(
+ "Unknown protocol: " + protoName);
+ }
+ // protocol supported but not the version that client wants
+ throw new RPC.VersionMismatch(protoName, clientVersion,
+ highest.version);
+ }
+ return impl;
+ }
+
+ @Override
+ /**
+ * This is a server side method, which is invoked over RPC. On success
+ * the return response has protobuf response payload. On failure, the
+ * exception name and the stack trace are returned in the response.
+ * See {@link HadoopRpcResponseProto}
+ *
+ * In this method there three types of exceptions possible and they are
+ * returned in response as follows.
+ *
+ * Exceptions encountered in this method that are returned
+ * as {@link RpcServerException}
+ * Exceptions thrown by the service is wrapped in ServiceException.
+ * In that this method returns in response the exception thrown by the
+ * service.
+ * Other exceptions thrown by the service. They are returned as
+ * it is.
+ *
+ */
+ public Writable call(RPC.Server server, String connectionProtocolName,
+ Writable writableRequest, long receiveTime) throws Exception {
+ RpcProtobufRequest request = (RpcProtobufRequest) writableRequest;
+ RequestHeaderProto rpcRequest = request.getRequestHeader();
+ String methodName = rpcRequest.getMethodName();
+
+ /**
+ * RPCs for a particular interface (ie protocol) are done using a
+ * IPC connection that is setup using rpcProxy.
+ * The rpcProxy's has a declared protocol name that is
+ * sent form client to server at connection time.
+ *
+ * Each Rpc call also sends a protocol name
+ * (called declaringClassprotocolName). This name is usually the same
+ * as the connection protocol name except in some cases.
+ * For example metaProtocols such ProtocolInfoProto which get info
+ * about the protocol reuse the connection but need to indicate that
+ * the actual protocol is different (i.e. the protocol is
+ * ProtocolInfoProto) since they reuse the connection; in this case
+ * the declaringClassProtocolName field is set to the ProtocolInfoProto.
+ */
+
+ String declaringClassProtoName =
+ rpcRequest.getDeclaringClassProtocolName();
+ long clientVersion = rpcRequest.getClientProtocolVersion();
+ return call(server, connectionProtocolName, request, receiveTime,
+ methodName, declaringClassProtoName, clientVersion);
+ }
+
+ protected Writable call(RPC.Server server, String connectionProtocolName,
+ RpcWritable.Buffer request, long receiveTime, String methodName,
+ String declaringClassProtoName, long clientVersion) throws Exception {
+ if (server.verbose)
+ LOG.info("Call: connectionProtocolName=" + connectionProtocolName +
+ ", method=" + methodName);
+
+ ProtoClassProtoImpl protocolImpl = getProtocolImpl(server,
+ declaringClassProtoName, clientVersion);
+ BlockingService service = (BlockingService) protocolImpl.protocolImpl;
+ MethodDescriptor methodDescriptor = service.getDescriptorForType()
+ .findMethodByName(methodName);
+ if (methodDescriptor == null) {
+ String msg = "Unknown method " + methodName + " called on "
+ + connectionProtocolName + " protocol.";
+ LOG.warn(msg);
+ throw new RpcNoSuchMethodException(msg);
+ }
+ Message prototype = service.getRequestPrototype(methodDescriptor);
+ Message param = request.getValue(prototype);
+
+ Message result;
+ Call currentCall = Server.getCurCall().get();
+ try {
+ server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
+ currentCallInfo.set(new CallInfo(server, methodName));
+ currentCall.setDetailedMetricsName(methodName);
+ result = service.callBlockingMethod(methodDescriptor, null, param);
+ // Check if this needs to be a deferred response,
+ // by checking the ThreadLocal callback being set
+ if (currentCallback.get() != null) {
+ currentCall.deferResponse();
+ currentCallback.set(null);
+ return null;
+ }
+ } catch (ServiceException e) {
+ Exception exception = (Exception) e.getCause();
+ currentCall.setDetailedMetricsName(
+ exception.getClass().getSimpleName());
+ throw (Exception) e.getCause();
+ } catch (Exception e) {
+ currentCall.setDetailedMetricsName(e.getClass().getSimpleName());
+ throw e;
+ } finally {
+ currentCallInfo.set(null);
+ }
+ return RpcWritable.wrap(result);
+ }
+ }
+ }
+
+ // htrace in the ipc layer creates the span name based on toString()
+ // which uses the rpc header. in the normal case we want to defer decoding
+ // the rpc header until needed by the rpc engine.
+ static class RpcProtobufRequest extends RpcWritable.Buffer {
+ private volatile RequestHeaderProto requestHeader;
+ private Message payload;
+
+ public RpcProtobufRequest() {
+ }
+
+ RpcProtobufRequest(RequestHeaderProto header, Message payload) {
+ this.requestHeader = header;
+ this.payload = payload;
+ }
+
+ RequestHeaderProto getRequestHeader() throws IOException {
+ if (getByteBuffer() != null && requestHeader == null) {
+ requestHeader = getValue(RequestHeaderProto.getDefaultInstance());
+ }
+ return requestHeader;
+ }
+
+ @Override
+ public void writeTo(ResponseBuffer out) throws IOException {
+ requestHeader.writeDelimitedTo(out);
+ if (payload != null) {
+ payload.writeDelimitedTo(out);
+ }
+ }
+
+ // this is used by htrace to name the span.
+ @Override
+ public String toString() {
+ try {
+ RequestHeaderProto header = getRequestHeader();
+ return header.getDeclaringClassProtocolName() + "." +
+ header.getMethodName();
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufRpcEngineCallback.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufRpcEngineCallback.java
new file mode 100644
index 000000000000..b76ca7d44fce
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtobufRpcEngineCallback.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import com.google.protobuf.Message;
+
+public interface ProtobufRpcEngineCallback {
+
+ public void setResponse(Message message);
+
+ public void error(Throwable t);
+
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolInfo.java
new file mode 100644
index 000000000000..439de87f6e9b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolInfo.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+
+/**
+ * The protocol name that is used when a client and server connect.
+ * By default the class name of the protocol interface is the protocol name.
+ *
+ * Why override the default name (i.e. the class name)?
+ * One use case overriding the default name (i.e. the class name) is when
+ * there are multiple implementations of the same protocol, each with say a
+ * different version/serialization.
+ * In Hadoop this is used to allow multiple server and client adapters
+ * for different versions of the same protocol service.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ProtocolInfo {
+ String protocolName(); // the name of the protocol (i.e. rpc service)
+ long protocolVersion() default -1; // default means not defined use old way
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolMetaInfoPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolMetaInfoPB.java
new file mode 100644
index 000000000000..8dc7138c2d78
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolMetaInfoPB.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.ProtocolInfoService;
+
+/**
+ * Protocol to get versions and signatures for supported protocols from the
+ * server.
+ *
+ * Note: This extends the protocolbuffer service based interface to
+ * add annotations.
+ */
+@ProtocolInfo(
+ protocolName = "org.apache.hadoop.ipc_.ProtocolMetaInfoPB",
+ protocolVersion = 1)
+public interface ProtocolMetaInfoPB extends
+ ProtocolInfoService.BlockingInterface {
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolMetaInfoServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolMetaInfoServerSideTranslatorPB.java
new file mode 100644
index 000000000000..1e07877325f0
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolMetaInfoServerSideTranslatorPB.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import org.apache.hadoop.ipc_.RPC.Server.VerProtocolImpl;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.GetProtocolVersionsRequestProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.GetProtocolVersionsResponseProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.ProtocolVersionProto;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+/**
+ * This class serves the requests for protocol versions and signatures by
+ * looking them up in the server registry.
+ */
+public class ProtocolMetaInfoServerSideTranslatorPB implements
+ ProtocolMetaInfoPB {
+
+ RPC.Server server;
+
+ public ProtocolMetaInfoServerSideTranslatorPB(RPC.Server server) {
+ this.server = server;
+ }
+
+ @Override
+ public GetProtocolVersionsResponseProto getProtocolVersions(
+ RpcController controller, GetProtocolVersionsRequestProto request)
+ throws ServiceException {
+ String protocol = request.getProtocol();
+ GetProtocolVersionsResponseProto.Builder builder =
+ GetProtocolVersionsResponseProto.newBuilder();
+ for (RPC.RpcKind r : RPC.RpcKind.values()) {
+ long[] versions;
+ try {
+ versions = getProtocolVersionForRpcKind(r, protocol);
+ } catch (ClassNotFoundException e) {
+ throw new ServiceException(e);
+ }
+ ProtocolVersionProto.Builder b = ProtocolVersionProto.newBuilder();
+ if (versions != null) {
+ b.setRpcKind(r.toString());
+ for (long v : versions) {
+ b.addVersions(v);
+ }
+ }
+ builder.addProtocolVersions(b.build());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public GetProtocolSignatureResponseProto getProtocolSignature(
+ RpcController controller, GetProtocolSignatureRequestProto request)
+ throws ServiceException {
+ GetProtocolSignatureResponseProto.Builder builder = GetProtocolSignatureResponseProto
+ .newBuilder();
+ String protocol = request.getProtocol();
+ String rpcKind = request.getRpcKind();
+ long[] versions;
+ try {
+ versions = getProtocolVersionForRpcKind(RPC.RpcKind.valueOf(rpcKind),
+ protocol);
+ } catch (ClassNotFoundException e1) {
+ throw new ServiceException(e1);
+ }
+ if (versions == null) {
+ return builder.build();
+ }
+ for (long v : versions) {
+ ProtocolSignatureProto.Builder sigBuilder = ProtocolSignatureProto
+ .newBuilder();
+ sigBuilder.setVersion(v);
+ try {
+ ProtocolSignature signature = ProtocolSignature.getProtocolSignature(
+ protocol, v);
+ for (int m : signature.getMethods()) {
+ sigBuilder.addMethods(m);
+ }
+ } catch (ClassNotFoundException e) {
+ throw new ServiceException(e);
+ }
+ builder.addProtocolSignature(sigBuilder.build());
+ }
+ return builder.build();
+ }
+
+ private long[] getProtocolVersionForRpcKind(RPC.RpcKind rpcKind,
+ String protocol) throws ClassNotFoundException {
+ Class protocolClass = Class.forName(protocol);
+ String protocolName = RPC.getProtocolName(protocolClass);
+ VerProtocolImpl[] vers = server.getSupportedProtocolVersions(rpcKind,
+ protocolName);
+ if (vers == null) {
+ return null;
+ }
+ long [] versions = new long[vers.length];
+ for (int i=0; i {
+ private Class protocol;
+ private T proxy;
+ private HashSet serverMethods = null;
+ final private boolean supportServerMethodCheck;
+ private boolean serverMethodsFetched = false;
+
+ /**
+ * Constructor
+ *
+ * @param protocol protocol class
+ * @param proxy its proxy
+ * @param supportServerMethodCheck If false proxy will never fetch server
+ * methods and isMethodSupported will always return true. If true,
+ * server methods will be fetched for the first call to
+ * isMethodSupported.
+ */
+ public ProtocolProxy(Class protocol, T proxy,
+ boolean supportServerMethodCheck) {
+ this.protocol = protocol;
+ this.proxy = proxy;
+ this.supportServerMethodCheck = supportServerMethodCheck;
+ }
+
+ private void fetchServerMethods(Method method) throws IOException {
+ long clientVersion;
+ clientVersion = RPC.getProtocolVersion(method.getDeclaringClass());
+ int clientMethodsHash = ProtocolSignature.getFingerprint(method
+ .getDeclaringClass().getMethods());
+ ProtocolSignature serverInfo = ((VersionedProtocol) proxy)
+ .getProtocolSignature(RPC.getProtocolName(protocol), clientVersion,
+ clientMethodsHash);
+ long serverVersion = serverInfo.getVersion();
+ if (serverVersion != clientVersion) {
+ throw new RPC.VersionMismatch(protocol.getName(), clientVersion,
+ serverVersion);
+ }
+ int[] serverMethodsCodes = serverInfo.getMethods();
+ if (serverMethodsCodes != null) {
+ serverMethods = new HashSet(serverMethodsCodes.length);
+ for (int m : serverMethodsCodes) {
+ this.serverMethods.add(Integer.valueOf(m));
+ }
+ }
+ serverMethodsFetched = true;
+ }
+
+ /*
+ * Get the proxy
+ */
+ public T getProxy() {
+ return proxy;
+ }
+
+ /**
+ * Check if a method is supported by the server or not.
+ *
+ * @param methodName a method's name in String format
+ * @param parameterTypes a method's parameter types
+ * @return true if the method is supported by the server
+ * @throws IOException raised on errors performing I/O.
+ */
+ public synchronized boolean isMethodSupported(String methodName,
+ Class... parameterTypes)
+ throws IOException {
+ if (!supportServerMethodCheck) {
+ return true;
+ }
+ Method method;
+ try {
+ method = protocol.getDeclaredMethod(methodName, parameterTypes);
+ } catch (SecurityException e) {
+ throw new IOException(e);
+ } catch (NoSuchMethodException e) {
+ throw new IOException(e);
+ }
+ if (!serverMethodsFetched) {
+ fetchServerMethods(method);
+ }
+ if (serverMethods == null) { // client & server have the same protocol
+ return true;
+ }
+ return serverMethods.contains(
+ Integer.valueOf(ProtocolSignature.getFingerprint(method)));
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolSignature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolSignature.java
new file mode 100644
index 000000000000..370002e3ad72
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolSignature.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+
+
+public class ProtocolSignature implements Writable {
+ static { // register a ctor
+ WritableFactories.setFactory
+ (ProtocolSignature.class,
+ new WritableFactory() {
+ @Override
+ public Writable newInstance() { return new ProtocolSignature(); }
+ });
+ }
+
+ private long version;
+ private int[] methods = null; // an array of method hash codes
+
+ /**
+ * default constructor
+ */
+ public ProtocolSignature() {
+ }
+
+ /**
+ * Constructor
+ *
+ * @param version server version
+ * @param methodHashcodes hash codes of the methods supported by server
+ */
+ public ProtocolSignature(long version, int[] methodHashcodes) {
+ this.version = version;
+ this.methods = methodHashcodes;
+ }
+
+ public long getVersion() {
+ return version;
+ }
+
+ public int[] getMethods() {
+ return methods;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ version = in.readLong();
+ boolean hasMethods = in.readBoolean();
+ if (hasMethods) {
+ int numMethods = in.readInt();
+ methods = new int[numMethods];
+ for (int i=0; i type : method.getParameterTypes()) {
+ hashcode = 31*hashcode ^ type.getName().hashCode();
+ }
+ return hashcode;
+ }
+
+ /**
+ * Convert an array of Method into an array of hash codes
+ *
+ * @param methods
+ * @return array of hash codes
+ */
+ private static int[] getFingerprints(Method[] methods) {
+ if (methods == null) {
+ return null;
+ }
+ int[] hashCodes = new int[methods.length];
+ for (int i = 0; i
+ PROTOCOL_FINGERPRINT_CACHE =
+ new HashMap();
+
+ public static void resetCache() {
+ PROTOCOL_FINGERPRINT_CACHE.clear();
+ }
+
+ /**
+ * Return a protocol's signature and finger print from cache
+ *
+ * @param protocol a protocol class
+ * @param serverVersion protocol version
+ * @return its signature and finger print
+ */
+ private static ProtocolSigFingerprint getSigFingerprint(
+ Class protocol, long serverVersion) {
+ String protocolName = RPC.getProtocolName(protocol);
+ synchronized (PROTOCOL_FINGERPRINT_CACHE) {
+ ProtocolSigFingerprint sig = PROTOCOL_FINGERPRINT_CACHE.get(protocolName);
+ if (sig == null) {
+ int[] serverMethodHashcodes = getFingerprints(protocol.getMethods());
+ sig = new ProtocolSigFingerprint(
+ new ProtocolSignature(serverVersion, serverMethodHashcodes),
+ getFingerprint(serverMethodHashcodes));
+ PROTOCOL_FINGERPRINT_CACHE.put(protocolName, sig);
+ }
+ return sig;
+ }
+ }
+
+ /**
+ * Get a server protocol's signature
+ *
+ * @param clientMethodsHashCode client protocol methods hashcode
+ * @param serverVersion server protocol version
+ * @param protocol protocol
+ * @return the server's protocol signature
+ */
+ public static ProtocolSignature getProtocolSignature(
+ int clientMethodsHashCode,
+ long serverVersion,
+ Class protocol) {
+ // try to get the finger print & signature from the cache
+ ProtocolSigFingerprint sig = getSigFingerprint(protocol, serverVersion);
+
+ // check if the client side protocol matches the one on the server side
+ if (clientMethodsHashCode == sig.fingerprint) {
+ return new ProtocolSignature(serverVersion, null); // null indicates a match
+ }
+
+ return sig.signature;
+ }
+
+ public static ProtocolSignature getProtocolSignature(String protocolName,
+ long version) throws ClassNotFoundException {
+ Class protocol = Class.forName(protocolName);
+ return getSigFingerprint(protocol, version).signature;
+ }
+
+ /**
+ * Get a server protocol's signature
+ *
+ * @param server server implementation
+ * @param protocol server protocol
+ * @param clientVersion client's version
+ * @param clientMethodsHash client's protocol's hash code
+ * @return the server protocol's signature
+ * @throws IOException if any error occurs
+ */
+ @SuppressWarnings("unchecked")
+ public static ProtocolSignature getProtocolSignature(VersionedProtocol server,
+ String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class inter;
+ try {
+ inter = (Class)Class.forName(protocol);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ long serverVersion = server.getProtocolVersion(protocol, clientVersion);
+ return ProtocolSignature.getProtocolSignature(
+ clientMethodsHash, serverVersion, inter);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolTranslator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolTranslator.java
new file mode 100644
index 000000000000..ea4ecb054d90
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProtocolTranslator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+
+/**
+ * An interface implemented by client-side protocol translators to get the
+ * underlying proxy object the translator is operating on.
+ */
+public interface ProtocolTranslator {
+
+ /**
+ * Return the proxy object underlying this protocol translator.
+ * @return the proxy object underlying this protocol translator.
+ */
+ public Object getUnderlyingProxyObject();
+
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProxyCombiner.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProxyCombiner.java
new file mode 100644
index 000000000000..7a2410dc00c6
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ProxyCombiner.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import com.google.common.base.Joiner;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
+import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.ipc_.Client.ConnectionId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * A utility class used to combine two protocol proxies.
+ * See {@link #combine(Class, Object...)}.
+ */
+public final class ProxyCombiner {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ProxyCombiner.class);
+
+ private ProxyCombiner() { }
+
+ /**
+ * Combine two or more proxies which together comprise a single proxy
+ * interface. This can be used for a protocol interface which {@code extends}
+ * multiple other protocol interfaces. The returned proxy will implement
+ * all of the methods of the combined proxy interface, delegating calls
+ * to which proxy implements that method. If multiple proxies implement the
+ * same method, the first in the list will be used for delegation.
+ *
+ *
This will check that every method on the combined interface is
+ * implemented by at least one of the supplied proxy objects.
+ *
+ * @param combinedProxyInterface The interface of the combined proxy.
+ * @param proxies The proxies which should be used as delegates.
+ * @param The type of the proxy that will be returned.
+ * @return The combined proxy.
+ */
+ @SuppressWarnings("unchecked")
+ public static T combine(Class combinedProxyInterface,
+ Object... proxies) {
+ methodLoop:
+ for (Method m : combinedProxyInterface.getMethods()) {
+ for (Object proxy : proxies) {
+ try {
+ proxy.getClass().getMethod(m.getName(), m.getParameterTypes());
+ continue methodLoop; // go to the next method
+ } catch (NoSuchMethodException nsme) {
+ // Continue to try the next proxy
+ }
+ }
+ throw new IllegalStateException("The proxies specified for "
+ + combinedProxyInterface + " do not cover method " + m);
+ }
+
+ InvocationHandler handler =
+ new CombinedProxyInvocationHandler(combinedProxyInterface, proxies);
+ return (T) Proxy.newProxyInstance(combinedProxyInterface.getClassLoader(),
+ new Class[] {combinedProxyInterface}, handler);
+ }
+
+ private static final class CombinedProxyInvocationHandler
+ implements RpcInvocationHandler {
+
+ private final Class proxyInterface;
+ private final Object[] proxies;
+
+ private CombinedProxyInvocationHandler(Class proxyInterface,
+ Object[] proxies) {
+ this.proxyInterface = proxyInterface;
+ this.proxies = proxies;
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args)
+ throws Throwable {
+ Exception lastException = null;
+ for (Object underlyingProxy : proxies) {
+ try {
+ return method.invoke(underlyingProxy, args);
+ } catch (IllegalAccessException|IllegalArgumentException e) {
+ lastException = e;
+ } catch (InvocationTargetException ite) {
+ throw ite.getCause();
+ }
+ }
+ // This shouldn't happen since the method coverage was verified in build()
+ LOG.error("BUG: Method {} was unable to be found on any of the "
+ + "underlying proxies for {}", method, proxy.getClass());
+ throw new IllegalArgumentException("Method " + method + " not supported",
+ lastException);
+ }
+
+ /**
+ * Since this is incapable of returning multiple connection IDs, simply
+ * return the first one. In most cases, the connection ID should be the same
+ * for all proxies.
+ */
+ @Override
+ public ConnectionId getConnectionId() {
+ return RPC.getConnectionIdForProxy(proxies[0]);
+ }
+
+ @Override
+ public String toString() {
+ return "CombinedProxy[" + proxyInterface.getSimpleName() + "]["
+ + Joiner.on(",").join(proxies) + "]";
+ }
+
+ @Override
+ public void close() throws IOException {
+ MultipleIOException.Builder exceptionBuilder =
+ new MultipleIOException.Builder();
+ for (Object proxy : proxies) {
+ if (proxy instanceof Closeable) {
+ try {
+ ((Closeable) proxy).close();
+ } catch (IOException ioe) {
+ exceptionBuilder.add(ioe);
+ }
+ }
+ }
+ if (!exceptionBuilder.isEmpty()) {
+ throw exceptionBuilder.build();
+ }
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RPC.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RPC.java
new file mode 100644
index 000000000000..2c544716f951
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RPC.java
@@ -0,0 +1,1168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Proxy;
+
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.net.NoRouteToHostException;
+import java.net.SocketTimeoutException;
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc_.Client.ConnectionId;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.ProtocolInfoService;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
+
+import com.google.protobuf.BlockingService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** A simple RPC mechanism.
+ *
+ * A protocol is a Java interface. All parameters and return types must
+ * be one of:
+ *
+ * a primitive type, boolean, byte,
+ * char, short, int, long,
+ * float, double, or void; or
+ *
+ * a {@link String}; or
+ *
+ * a {@link Writable}; or
+ *
+ * an array of the above types
+ *
+ * All methods in the protocol should throw only IOException. No field data of
+ * the protocol instance is transmitted.
+ */
+public class RPC {
+ final static int RPC_SERVICE_CLASS_DEFAULT = 0;
+ public enum RpcKind {
+ RPC_BUILTIN ((short) 1), // Used for built in calls by tests
+ RPC_WRITABLE ((short) 2), // Use WritableRpcEngine
+ RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
+ final static short MAX_SIZE = RPC_PROTOCOL_BUFFER.value; // used for array size
+ private final short value;
+
+ RpcKind(short val) {
+ this.value = val;
+ }
+ }
+
+ interface RpcInvoker {
+ /**
+ * Process a client call on the server side
+ * @param server the server within whose context this rpc call is made
+ * @param protocol - the protocol name (the class of the client proxy
+ * used to make calls to the rpc server.
+ * @param rpcRequest - deserialized
+ * @param receiveTime time at which the call received (for metrics)
+ * @return the call's return
+ * @throws IOException
+ **/
+ public Writable call(Server server, String protocol,
+ Writable rpcRequest, long receiveTime) throws Exception ;
+ }
+
+ static final Logger LOG = LoggerFactory.getLogger(RPC.class);
+
+ /**
+ * Get all superInterfaces that extend VersionedProtocol
+ * @param childInterfaces
+ * @return the super interfaces that extend VersionedProtocol
+ */
+ static Class[] getSuperInterfaces(Class[] childInterfaces) {
+ List> allInterfaces = new ArrayList>();
+
+ for (Class childInterface : childInterfaces) {
+ if (VersionedProtocol.class.isAssignableFrom(childInterface)) {
+ allInterfaces.add(childInterface);
+ allInterfaces.addAll(
+ Arrays.asList(
+ getSuperInterfaces(childInterface.getInterfaces())));
+ } else {
+ LOG.warn("Interface " + childInterface +
+ " ignored because it does not extend VersionedProtocol");
+ }
+ }
+ return allInterfaces.toArray(new Class[allInterfaces.size()]);
+ }
+
+ /**
+ * Get all interfaces that the given protocol implements or extends
+ * which are assignable from VersionedProtocol.
+ */
+ static Class[] getProtocolInterfaces(Class protocol) {
+ Class[] interfaces = protocol.getInterfaces();
+ return getSuperInterfaces(interfaces);
+ }
+
+ /**
+ * Get the protocol name.
+ * If the protocol class has a ProtocolAnnotation, then get the protocol
+ * name from the annotation; otherwise the class name is the protocol name.
+ *
+ * @param protocol input protocol.
+ * @return protocol name.
+ */
+ static public String getProtocolName(Class protocol) {
+ if (protocol == null) {
+ return null;
+ }
+ ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
+ return (anno == null) ? protocol.getName() : anno.protocolName();
+ }
+
+ /**
+ * Get the protocol version from protocol class.
+ * If the protocol class has a ProtocolAnnotation,
+ * then get the protocol version from the annotation;
+ * otherwise get it from the versionID field of the protocol class.
+ *
+ * @param protocol input protocol.
+ * @return ProtocolVersion.
+ */
+ static public long getProtocolVersion(Class protocol) {
+ if (protocol == null) {
+ throw new IllegalArgumentException("Null protocol");
+ }
+ long version;
+ ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
+ if (anno != null) {
+ version = anno.protocolVersion();
+ if (version != -1)
+ return version;
+ }
+ try {
+ Field versionField = protocol.getField("versionID");
+ versionField.setAccessible(true);
+ return versionField.getLong(protocol);
+ } catch (NoSuchFieldException ex) {
+ throw new RuntimeException(ex);
+ } catch (IllegalAccessException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private RPC() {} // no public ctor
+
+ // cache of RpcEngines by protocol
+ private static final Map,RpcEngine> PROTOCOL_ENGINES
+ = new HashMap,RpcEngine>();
+
+ private static final String ENGINE_PROP = "rpc.engine";
+
+ /**
+ * Set a protocol to use a non-default RpcEngine if one
+ * is not specified in the configuration.
+ * @param conf configuration to use
+ * @param protocol the protocol interface
+ * @param engine the RpcEngine impl
+ */
+ public static void setProtocolEngine(Configuration conf,
+ Class protocol, Class engine) {
+ if (conf.get(ENGINE_PROP+"."+protocol.getName()) == null) {
+ conf.setClass(ENGINE_PROP+"."+protocol.getName(), engine,
+ RpcEngine.class);
+ }
+ }
+
+ // return the RpcEngine configured to handle a protocol
+ static synchronized RpcEngine getProtocolEngine(Class protocol,
+ Configuration conf) {
+ RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
+ if (engine == null) {
+ Class impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
+ WritableRpcEngine.class);
+ engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
+ PROTOCOL_ENGINES.put(protocol, engine);
+ }
+ return engine;
+ }
+
+ /**
+ * A version mismatch for the RPC protocol.
+ */
+ public static class VersionMismatch extends RpcServerException {
+ private static final long serialVersionUID = 0;
+
+ private String interfaceName;
+ private long clientVersion;
+ private long serverVersion;
+
+ /**
+ * Create a version mismatch exception
+ * @param interfaceName the name of the protocol mismatch
+ * @param clientVersion the client's version of the protocol
+ * @param serverVersion the server's version of the protocol
+ */
+ public VersionMismatch(String interfaceName, long clientVersion,
+ long serverVersion) {
+ super("Protocol " + interfaceName + " version mismatch. (client = " +
+ clientVersion + ", server = " + serverVersion + ")");
+ this.interfaceName = interfaceName;
+ this.clientVersion = clientVersion;
+ this.serverVersion = serverVersion;
+ }
+
+ /**
+ * Get the interface name
+ * @return the java class name
+ * (eg. org.apache.hadoop.mapred.InterTrackerProtocol)
+ */
+ public String getInterfaceName() {
+ return interfaceName;
+ }
+
+ /**
+ * @return Get the client's preferred version.
+ */
+ public long getClientVersion() {
+ return clientVersion;
+ }
+
+ /**
+ * @return Get the server's agreed to version.
+ */
+ public long getServerVersion() {
+ return serverVersion;
+ }
+ /**
+ * get the rpc status corresponding to this exception
+ */
+ public RpcStatusProto getRpcStatusProto() {
+ return RpcStatusProto.ERROR;
+ }
+
+ /**
+ * get the detailed rpc status corresponding to this exception
+ */
+ public RpcErrorCodeProto getRpcErrorCodeProto() {
+ return RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH;
+ }
+ }
+
+ /**
+ * Get a proxy connection to a remote server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @return the proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static T waitForProxy(
+ Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ Configuration conf
+ ) throws IOException {
+ return waitForProtocolProxy(protocol, clientVersion, addr, conf).getProxy();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @return the protocol proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static ProtocolProxy waitForProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ Configuration conf) throws IOException {
+ return waitForProtocolProxy(
+ protocol, clientVersion, addr, conf, Long.MAX_VALUE);
+ }
+
+ /**
+ * Get a proxy connection to a remote server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @param connTimeout time in milliseconds before giving up
+ * @return the proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static T waitForProxy(Class protocol, long clientVersion,
+ InetSocketAddress addr, Configuration conf,
+ long connTimeout) throws IOException {
+ return waitForProtocolProxy(protocol, clientVersion, addr,
+ conf, connTimeout).getProxy();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @param connTimeout time in milliseconds before giving up
+ * @return the protocol proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static ProtocolProxy waitForProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf,
+ long connTimeout) throws IOException {
+ return waitForProtocolProxy(protocol, clientVersion, addr, conf,
+ getRpcTimeout(conf), null, connTimeout);
+ }
+
+ /**
+ * Get a proxy connection to a remote server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @param rpcTimeout timeout for each RPC
+ * @param timeout time in milliseconds before giving up
+ * @return the proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static T waitForProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf,
+ int rpcTimeout,
+ long timeout) throws IOException {
+ return waitForProtocolProxy(protocol, clientVersion, addr,
+ conf, rpcTimeout, null, timeout).getProxy();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param Generics Type.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @param rpcTimeout timeout for each RPC
+ * @param connectionRetryPolicy input connectionRetryPolicy.
+ * @param timeout time in milliseconds before giving up
+ * @return the proxy
+ * @throws IOException if the far end through a RemoteException.
+ */
+ public static ProtocolProxy waitForProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf,
+ int rpcTimeout,
+ RetryPolicy connectionRetryPolicy,
+ long timeout) throws IOException {
+ long startTime = Time.now();
+ IOException ioe;
+ while (true) {
+ try {
+ return getProtocolProxy(protocol, clientVersion, addr,
+ UserGroupInformation.getCurrentUser(), conf, NetUtils
+ .getDefaultSocketFactory(conf), rpcTimeout, connectionRetryPolicy);
+ } catch(ConnectException se) { // namenode has not been started
+ LOG.info("Server at " + addr + " not available yet, Zzzzz...");
+ ioe = se;
+ } catch(SocketTimeoutException te) { // namenode is busy
+ LOG.info("Problem connecting to server: " + addr);
+ ioe = te;
+ } catch(NoRouteToHostException nrthe) { // perhaps a VIP is failing over
+ LOG.info("No route to host for server: " + addr);
+ ioe = nrthe;
+ }
+ // check if timed out
+ if (Time.now()-timeout >= startTime) {
+ throw ioe;
+ }
+
+ if (Thread.currentThread().isInterrupted()) {
+ // interrupted during some IO; this may not have been caught
+ throw new InterruptedIOException("Interrupted waiting for the proxy");
+ }
+
+ // wait for retry
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw (IOException) new InterruptedIOException(
+ "Interrupted waiting for the proxy").initCause(ioe);
+ }
+ }
+ }
+
+ /**
+ * Construct a client-side proxy object that implements the named protocol,
+ * talking to a server at the named address.
+ * @param Generics Type T.
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param addr input addr.
+ * @param conf input Configuration.
+ * @param factory input factory.
+ * @throws IOException raised on errors performing I/O.
+ * @return proxy.
+ */
+ public static T getProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf,
+ SocketFactory factory) throws IOException {
+ return getProtocolProxy(
+ protocol, clientVersion, addr, conf, factory).getProxy();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param conf configuration to use
+ * @param factory socket factory
+ * @return the protocol proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf,
+ SocketFactory factory) throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ return getProtocolProxy(protocol, clientVersion, addr, ugi, conf, factory);
+ }
+
+ /**
+ * Construct a client-side proxy object that implements the named protocol,
+ * talking to a server at the named address.
+ *
+ * @param Generics Type T.
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param addr input addr.
+ * @param ticket input tocket.
+ * @param conf input conf.
+ * @param factory input factory.
+ * @return the protocol proxy.
+ * @throws IOException raised on errors performing I/O.
+ *
+ */
+ public static T getProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory) throws IOException {
+ return getProtocolProxy(
+ protocol, clientVersion, addr, ticket, conf, factory).getProxy();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server
+ *
+ * @param Generics Type T.
+ * @param protocol protocol class
+ * @param clientVersion client version
+ * @param addr remote address
+ * @param ticket user group information
+ * @param conf configuration to use
+ * @param factory socket factory
+ * @return the protocol proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory) throws IOException {
+ return getProtocolProxy(protocol, clientVersion, addr, ticket, conf,
+ factory, getRpcTimeout(conf), null);
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param Generics Type T
+ * @param protocol protocol class
+ * @param clientVersion client's version
+ * @param connId client connection identifier
+ * @param conf configuration
+ * @param factory socket factory
+ * @return the protocol proxy
+ * @throws IOException if the far end through a RemoteException
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion, ConnectionId connId, Configuration conf,
+ SocketFactory factory) throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SaslRpcServer.init(conf);
+ }
+ return getProtocolEngine(protocol, conf).getProxy(
+ protocol, clientVersion, connId, conf, factory);
+ }
+
+ /**
+ * Construct a client-side proxy that implements the named protocol,
+ * talking to a server at the named address.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol
+ * @param clientVersion client's version
+ * @param addr server address
+ * @param ticket security ticket
+ * @param conf configuration
+ * @param factory socket factory
+ * @param rpcTimeout max time for each rpc; 0 means no timeout
+ * @return the proxy
+ * @throws IOException if any error occurs
+ */
+ public static T getProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory,
+ int rpcTimeout) throws IOException {
+ return getProtocolProxy(protocol, clientVersion, addr, ticket,
+ conf, factory, rpcTimeout, null).getProxy();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol
+ * @param clientVersion client's version
+ * @param addr server address
+ * @param ticket security ticket
+ * @param conf configuration
+ * @param factory socket factory
+ * @param rpcTimeout max time for each rpc; 0 means no timeout
+ * @param connectionRetryPolicy retry policy
+ * @return the proxy
+ * @throws IOException if any error occurs
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory,
+ int rpcTimeout,
+ RetryPolicy connectionRetryPolicy) throws IOException {
+ return getProtocolProxy(protocol, clientVersion, addr, ticket,
+ conf, factory, rpcTimeout, connectionRetryPolicy, null);
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param Generics Type T.
+ * @param protocol protocol
+ * @param clientVersion client's version
+ * @param addr server address
+ * @param ticket security ticket
+ * @param conf configuration
+ * @param factory socket factory
+ * @param rpcTimeout max time for each rpc; 0 means no timeout
+ * @param connectionRetryPolicy retry policy
+ * @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ * a secure client falls back to simple auth
+ * @return the proxy
+ * @throws IOException if any error occurs
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory,
+ int rpcTimeout,
+ RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth)
+ throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SaslRpcServer.init(conf);
+ }
+ return getProtocolEngine(protocol, conf).getProxy(protocol, clientVersion,
+ addr, ticket, conf, factory, rpcTimeout, connectionRetryPolicy,
+ fallbackToSimpleAuth, null);
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server.
+ *
+ * @param protocol protocol
+ * @param clientVersion client's version
+ * @param addr server address
+ * @param ticket security ticket
+ * @param conf configuration
+ * @param factory socket factory
+ * @param rpcTimeout max time for each rpc; 0 means no timeout
+ * @param connectionRetryPolicy retry policy
+ * @param fallbackToSimpleAuth set to true or false during calls to indicate
+ * if a secure client falls back to simple auth
+ * @param alignmentContext state alignment context
+ * @param Generics Type T.
+ * @return the proxy
+ * @throws IOException if any error occurs
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory,
+ int rpcTimeout,
+ RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth,
+ AlignmentContext alignmentContext)
+ throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SaslRpcServer.init(conf);
+ }
+ return getProtocolEngine(protocol, conf).getProxy(protocol, clientVersion,
+ addr, ticket, conf, factory, rpcTimeout, connectionRetryPolicy,
+ fallbackToSimpleAuth, alignmentContext);
+ }
+
+ /**
+ * Construct a client-side proxy object with the default SocketFactory.
+ *
+ * @param Generics Type T.
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param addr input addr.
+ * @param conf input Configuration.
+ * @return a proxy instance
+ * @throws IOException if the thread is interrupted.
+ */
+ public static T getProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf)
+ throws IOException {
+
+ return getProtocolProxy(protocol, clientVersion, addr, conf).getProxy();
+ }
+
+ /**
+ * @return Returns the server address for a given proxy.
+ * @param proxy input proxy.
+ */
+ public static InetSocketAddress getServerAddress(Object proxy) {
+ return getConnectionIdForProxy(proxy).getAddress();
+ }
+
+ /**
+ * Return the connection ID of the given object. If the provided object is in
+ * fact a protocol translator, we'll get the connection ID of the underlying
+ * proxy object.
+ *
+ * @param proxy the proxy object to get the connection ID of.
+ * @return the connection ID for the provided proxy object.
+ */
+ public static ConnectionId getConnectionIdForProxy(Object proxy) {
+ if (proxy instanceof ProtocolTranslator) {
+ proxy = ((ProtocolTranslator)proxy).getUnderlyingProxyObject();
+ }
+ RpcInvocationHandler inv = (RpcInvocationHandler) Proxy
+ .getInvocationHandler(proxy);
+ return inv.getConnectionId();
+ }
+
+ /**
+ * Get a protocol proxy that contains a proxy connection to a remote server
+ * and a set of methods that are supported by the server
+ *
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param addr input addr.
+ * @param conf input configuration.
+ * @param Generics Type T.
+ * @return a protocol proxy
+ * @throws IOException if the thread is interrupted.
+ */
+ public static ProtocolProxy getProtocolProxy(Class protocol,
+ long clientVersion,
+ InetSocketAddress addr, Configuration conf)
+ throws IOException {
+
+ return getProtocolProxy(protocol, clientVersion, addr, conf, NetUtils
+ .getDefaultSocketFactory(conf));
+ }
+
+ /**
+ * Stop the proxy. Proxy must either implement {@link Closeable} or must have
+ * associated {@link RpcInvocationHandler}.
+ *
+ * @param proxy
+ * the RPC proxy object to be stopped
+ * @throws IllegalArgumentException
+ * if the proxy does not implement {@link Closeable} interface or
+ * does not have closeable {@link InvocationHandler}
+ */
+ public static void stopProxy(Object proxy) {
+ if (proxy == null) {
+ throw new IllegalArgumentException(
+ "Cannot close proxy since it is null");
+ }
+ try {
+ if (proxy instanceof Closeable) {
+ ((Closeable) proxy).close();
+ return;
+ } else {
+ InvocationHandler handler = Proxy.getInvocationHandler(proxy);
+ if (handler instanceof Closeable) {
+ ((Closeable) handler).close();
+ return;
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Closing proxy or invocation handler caused exception", e);
+ } catch (IllegalArgumentException e) {
+ LOG.error("RPC.stopProxy called on non proxy: class=" + proxy.getClass().getName(), e);
+ }
+
+ // If you see this error on a mock object in a unit test you're
+ // developing, make sure to use MockitoUtil.mockProtocol() to
+ // create your mock.
+ throw new IllegalArgumentException(
+ "Cannot close proxy - is not Closeable or "
+ + "does not provide closeable invocation handler "
+ + proxy.getClass());
+ }
+ /**
+ * Get the RPC time from configuration;
+ * If not set in the configuration, return the default value.
+ *
+ * @param conf Configuration
+ * @return the RPC timeout (ms)
+ */
+ public static int getRpcTimeout(Configuration conf) {
+ return conf.getInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_DEFAULT);
+ }
+
+ /**
+ * Class to construct instances of RPC server with specific options.
+ */
+ public static class Builder {
+ private Class protocol = null;
+ private Object instance = null;
+ private String bindAddress = "0.0.0.0";
+ private int port = 0;
+ private int numHandlers = 1;
+ private int numReaders = -1;
+ private int queueSizePerHandler = -1;
+ private boolean verbose = false;
+ private final Configuration conf;
+ private SecretManager secretManager = null;
+ private String portRangeConfig = null;
+ private AlignmentContext alignmentContext = null;
+
+ public Builder(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /** Mandatory field */
+ public Builder setProtocol(Class protocol) {
+ this.protocol = protocol;
+ return this;
+ }
+
+ /** Mandatory field */
+ public Builder setInstance(Object instance) {
+ this.instance = instance;
+ return this;
+ }
+
+ /** Default: 0.0.0.0 */
+ public Builder setBindAddress(String bindAddress) {
+ this.bindAddress = bindAddress;
+ return this;
+ }
+
+ /** Default: 0 */
+ public Builder setPort(int port) {
+ this.port = port;
+ return this;
+ }
+
+ /** Default: 1 */
+ public Builder setNumHandlers(int numHandlers) {
+ this.numHandlers = numHandlers;
+ return this;
+ }
+
+ /**
+ * @return Default: -1.
+ * @param numReaders input numReaders.
+ * @deprecated call {@link #setNumReaders(int value)} instead.
+ */
+ @Deprecated
+ public Builder setnumReaders(int numReaders) {
+ this.numReaders = numReaders;
+ return this;
+ }
+
+ /**
+ * Set the number of reader threads.
+ *
+ * @return this builder.
+ * @param value input numReaders.
+ * @since HADOOP-18625.
+ */
+ public Builder setNumReaders(int value) {
+ this.numReaders = value;
+ return this;
+ }
+
+ /** Default: -1 */
+ public Builder setQueueSizePerHandler(int queueSizePerHandler) {
+ this.queueSizePerHandler = queueSizePerHandler;
+ return this;
+ }
+
+ /** Default: false */
+ public Builder setVerbose(boolean verbose) {
+ this.verbose = verbose;
+ return this;
+ }
+
+ /** Default: null */
+ public Builder setSecretManager(
+ SecretManager secretManager) {
+ this.secretManager = secretManager;
+ return this;
+ }
+
+ /** Default: null */
+ public Builder setPortRangeConfig(String portRangeConfig) {
+ this.portRangeConfig = portRangeConfig;
+ return this;
+ }
+
+ /** Default: null */
+ public Builder setAlignmentContext(AlignmentContext alignmentContext) {
+ this.alignmentContext = alignmentContext;
+ return this;
+ }
+
+ /**
+ * Build the RPC Server.
+ * @throws IOException on error
+ * @throws IllegalArgumentException when mandatory fields are not set
+ */
+ public Server build() throws IOException, IllegalArgumentException {
+ if (this.conf == null) {
+ throw new IllegalArgumentException("conf is not set");
+ }
+ if (this.protocol == null) {
+ throw new IllegalArgumentException("protocol is not set");
+ }
+ if (this.instance == null) {
+ throw new IllegalArgumentException("instance is not set");
+ }
+
+ return getProtocolEngine(this.protocol, this.conf).getServer(
+ this.protocol, this.instance, this.bindAddress, this.port,
+ this.numHandlers, this.numReaders, this.queueSizePerHandler,
+ this.verbose, this.conf, this.secretManager, this.portRangeConfig,
+ this.alignmentContext);
+ }
+ }
+
+ /** An RPC Server. */
+ public abstract static class Server extends org.apache.hadoop.ipc_.Server {
+
+ boolean verbose;
+
+ private static final Pattern COMPLEX_SERVER_NAME_PATTERN =
+ Pattern.compile("(?:[^\\$]*\\$)*([A-Za-z][^\\$]+)(?:\\$\\d+)?");
+
+ /**
+ * Get a meaningful and short name for a server based on a java class.
+ *
+ * The rules are defined to support the current naming schema of the
+ * generated protobuf classes where the final class usually an anonymous
+ * inner class of an inner class.
+ *
+ * 1. For simple classes it returns with the simple name of the classes
+ * (with the name without package name)
+ *
+ * 2. For inner classes, this is the simple name of the inner class.
+ *
+ * 3. If it is an Object created from a class factory
+ * E.g., org.apache.hadoop.ipc_.TestRPC$TestClass$2
+ * this method returns parent class TestClass.
+ *
+ * 4. If it is an anonymous class E.g., 'org.apache.hadoop.ipc_.TestRPC$10'
+ * serverNameFromClass returns parent class TestRPC.
+ *
+ *
+ */
+ static String serverNameFromClass(Class clazz) {
+ String name = clazz.getName();
+ String[] names = clazz.getName().split("\\.", -1);
+ if (names != null && names.length > 0) {
+ name = names[names.length - 1];
+ }
+ Matcher matcher = COMPLEX_SERVER_NAME_PATTERN.matcher(name);
+ if (matcher.find()) {
+ return matcher.group(1);
+ } else {
+ return name;
+ }
+ }
+
+ /**
+ * Store a map of protocol and version to its implementation
+ */
+ /**
+ * The key in Map
+ */
+ static class ProtoNameVer {
+ final String protocol;
+ final long version;
+ ProtoNameVer(String protocol, long ver) {
+ this.protocol = protocol;
+ this.version = ver;
+ }
+ @Override
+ public boolean equals(Object o) {
+ if (o == null)
+ return false;
+ if (this == o)
+ return true;
+ if (! (o instanceof ProtoNameVer))
+ return false;
+ ProtoNameVer pv = (ProtoNameVer) o;
+ return ((pv.protocol.equals(this.protocol)) &&
+ (pv.version == this.version));
+ }
+ @Override
+ public int hashCode() {
+ return protocol.hashCode() * 37 + (int) version;
+ }
+ }
+
+ /**
+ * The value in map
+ */
+ static class ProtoClassProtoImpl {
+ final Class protocolClass;
+ final Object protocolImpl;
+ ProtoClassProtoImpl(Class protocolClass, Object protocolImpl) {
+ this.protocolClass = protocolClass;
+ this.protocolImpl = protocolImpl;
+ }
+ }
+
+ ArrayList> protocolImplMapArray =
+ new ArrayList>(RpcKind.MAX_SIZE);
+
+ @SuppressWarnings("checkstyle:Indentation")
+ Map getProtocolImplMap(RPC.RpcKind rpcKind) {
+ if (protocolImplMapArray.size() == 0) {// initialize for all rpc kinds
+ for (int i = 0; i < RpcKind.MAX_SIZE; ++i) {
+ protocolImplMapArray.add(
+ new HashMap(10));
+ }
+ }
+ return protocolImplMapArray.get(rpcKind.ordinal());
+ }
+
+ // Register protocol and its impl for rpc calls
+ void registerProtocolAndImpl(RpcKind rpcKind, Class protocolClass,
+ Object protocolImpl) {
+ String protocolName = RPC.getProtocolName(protocolClass);
+ long version;
+
+
+ try {
+ version = RPC.getProtocolVersion(protocolClass);
+ } catch (Exception ex) {
+ LOG.warn("Protocol " + protocolClass +
+ " NOT registered as cannot get protocol version ");
+ return;
+ }
+
+
+ getProtocolImplMap(rpcKind).put(new ProtoNameVer(protocolName, version),
+ new ProtoClassProtoImpl(protocolClass, protocolImpl));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RpcKind = " + rpcKind + " Protocol Name = " + protocolName +
+ " version=" + version +
+ " ProtocolImpl=" + protocolImpl.getClass().getName() +
+ " protocolClass=" + protocolClass.getName());
+ }
+ String client = SecurityUtil.getClientPrincipal(protocolClass, getConf());
+ if (client != null) {
+ // notify the server's rpc scheduler that the protocol user has
+ // highest priority. the scheduler should exempt the user from
+ // priority calculations.
+ try {
+ setPriorityLevel(UserGroupInformation.createRemoteUser(client), -1);
+ } catch (Exception ex) {
+ LOG.warn("Failed to set scheduling priority for " + client, ex);
+ }
+ }
+ }
+
+ static class VerProtocolImpl {
+ final long version;
+ final ProtoClassProtoImpl protocolTarget;
+ VerProtocolImpl(long ver, ProtoClassProtoImpl protocolTarget) {
+ this.version = ver;
+ this.protocolTarget = protocolTarget;
+ }
+ }
+
+ VerProtocolImpl[] getSupportedProtocolVersions(RPC.RpcKind rpcKind,
+ String protocolName) {
+ VerProtocolImpl[] resultk =
+ new VerProtocolImpl[getProtocolImplMap(rpcKind).size()];
+ int i = 0;
+ for (Map.Entry pv :
+ getProtocolImplMap(rpcKind).entrySet()) {
+ if (pv.getKey().protocol.equals(protocolName)) {
+ resultk[i++] =
+ new VerProtocolImpl(pv.getKey().version, pv.getValue());
+ }
+ }
+ if (i == 0) {
+ return null;
+ }
+ VerProtocolImpl[] result = new VerProtocolImpl[i];
+ System.arraycopy(resultk, 0, result, 0, i);
+ return result;
+ }
+
+ VerProtocolImpl getHighestSupportedProtocol(RpcKind rpcKind,
+ String protocolName) {
+ Long highestVersion = 0L;
+ ProtoClassProtoImpl highest = null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Size of protoMap for " + rpcKind + " ="
+ + getProtocolImplMap(rpcKind).size());
+ }
+ for (Map.Entry pv :
+ getProtocolImplMap(rpcKind).entrySet()) {
+ if (pv.getKey().protocol.equals(protocolName)) {
+ if ((highest == null) || (pv.getKey().version > highestVersion)) {
+ highest = pv.getValue();
+ highestVersion = pv.getKey().version;
+ }
+ }
+ }
+ if (highest == null) {
+ return null;
+ }
+ return new VerProtocolImpl(highestVersion, highest);
+ }
+
+ protected Server(String bindAddress, int port,
+ Class paramClass, int handlerCount,
+ int numReaders, int queueSizePerHandler,
+ Configuration conf, String serverName,
+ SecretManager secretManager,
+ String portRangeConfig) throws IOException {
+ super(bindAddress, port, paramClass, handlerCount, numReaders, queueSizePerHandler,
+ conf, serverName, secretManager, portRangeConfig);
+ initProtocolMetaInfo(conf);
+ }
+
+ private void initProtocolMetaInfo(Configuration conf) {
+ RPC.setProtocolEngine(conf, ProtocolMetaInfoPB.class,
+ ProtobufRpcEngine.class);
+ ProtocolMetaInfoServerSideTranslatorPB xlator =
+ new ProtocolMetaInfoServerSideTranslatorPB(this);
+ BlockingService protocolInfoBlockingService = ProtocolInfoService
+ .newReflectiveBlockingService(xlator);
+ addProtocol(RpcKind.RPC_PROTOCOL_BUFFER, ProtocolMetaInfoPB.class,
+ protocolInfoBlockingService);
+ }
+
+ /**
+ * Add a protocol to the existing server.
+ * @param rpcKind - input rpcKind
+ * @param protocolClass - the protocol class
+ * @param protocolImpl - the impl of the protocol that will be called
+ * @return the server (for convenience)
+ */
+ public Server addProtocol(RpcKind rpcKind, Class protocolClass,
+ Object protocolImpl) {
+ registerProtocolAndImpl(rpcKind, protocolClass, protocolImpl);
+ return this;
+ }
+
+ @Override
+ public Writable call(RPC.RpcKind rpcKind, String protocol,
+ Writable rpcRequest, long receiveTime) throws Exception {
+ return getRpcInvoker(rpcKind).call(this, protocol, rpcRequest,
+ receiveTime);
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshCallQueueProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshCallQueueProtocol.java
new file mode 100644
index 000000000000..b5348c8dfbcb
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshCallQueueProtocol.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * Protocol which is used to refresh the call queue in use currently.
+ */
+@KerberosInfo(
+ serverPrincipal=CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY)
+public interface RefreshCallQueueProtocol {
+
+ /**
+ * Version 1: Initial version
+ */
+ public static final long versionID = 1L;
+
+ /**
+ * Refresh the callqueue.
+ * @throws IOException raised on errors performing I/O.
+ */
+ @Idempotent
+ void refreshCallQueue() throws IOException;
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshHandler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshHandler.java
new file mode 100644
index 000000000000..ededbcb9b9b4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshHandler.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+
+/**
+ * Used to registry custom methods to refresh at runtime.
+ */
+public interface RefreshHandler {
+ /**
+ * Implement this method to accept refresh requests from the administrator.
+ * @param identifier is the identifier you registered earlier
+ * @param args contains a list of string args from the administrator
+ * @return a RefreshResponse
+ */
+ RefreshResponse handleRefresh(String identifier, String[] args);
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshRegistry.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshRegistry.java
new file mode 100644
index 000000000000..3f39f0680a1c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshRegistry.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to registry custom methods to refresh at runtime.
+ * Each identifier maps to one or more RefreshHandlers.
+ */
+public class RefreshRegistry {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(RefreshRegistry.class);
+
+ // Used to hold singleton instance
+ private static class RegistryHolder {
+ @SuppressWarnings("All")
+ public static RefreshRegistry registry = new RefreshRegistry();
+ }
+
+ // Singleton access
+ public static RefreshRegistry defaultRegistry() {
+ return RegistryHolder.registry;
+ }
+
+ private final Multimap handlerTable;
+
+ public RefreshRegistry() {
+ handlerTable = HashMultimap.create();
+ }
+
+ /**
+ * Registers an object as a handler for a given identity.
+ * Note: will prevent handler from being GC'd, object should unregister itself
+ * when done
+ * @param identifier a unique identifier for this resource,
+ * such as org.apache.hadoop.blacklist
+ * @param handler the object to register
+ */
+ public synchronized void register(String identifier, RefreshHandler handler) {
+ if (identifier == null) {
+ throw new NullPointerException("Identifier cannot be null");
+ }
+ handlerTable.put(identifier, handler);
+ }
+
+ /**
+ * Remove the registered object for a given identity.
+ * @param identifier the resource to unregister
+ * @param handler input handler.
+ * @return the true if removed
+ */
+ public synchronized boolean unregister(String identifier, RefreshHandler handler) {
+ return handlerTable.remove(identifier, handler);
+ }
+
+ public synchronized void unregisterAll(String identifier) {
+ handlerTable.removeAll(identifier);
+ }
+
+ /**
+ * Lookup the responsible handler and return its result.
+ * This should be called by the RPC server when it gets a refresh request.
+ * @param identifier the resource to refresh
+ * @param args the arguments to pass on, not including the program name
+ * @throws IllegalArgumentException on invalid identifier
+ * @return the response from the appropriate handler
+ */
+ public synchronized Collection dispatch(String identifier, String[] args) {
+ Collection handlers = handlerTable.get(identifier);
+
+ if (handlers.size() == 0) {
+ String msg = "Identifier '" + identifier +
+ "' does not exist in RefreshRegistry. Valid options are: " +
+ Joiner.on(", ").join(handlerTable.keySet());
+
+ throw new IllegalArgumentException(msg);
+ }
+
+ ArrayList responses =
+ new ArrayList(handlers.size());
+
+ // Dispatch to each handler and store response
+ for(RefreshHandler handler : handlers) {
+ RefreshResponse response;
+
+ // Run the handler
+ try {
+ response = handler.handleRefresh(identifier, args);
+ if (response == null) {
+ throw new NullPointerException("Handler returned null.");
+ }
+
+ LOG.info(handlerName(handler) + " responds to '" + identifier +
+ "', says: '" + response.getMessage() + "', returns " +
+ response.getReturnCode());
+ } catch (Exception e) {
+ response = new RefreshResponse(-1, e.getLocalizedMessage());
+ }
+
+ response.setSenderName(handlerName(handler));
+ responses.add(response);
+ }
+
+ return responses;
+ }
+
+ private String handlerName(RefreshHandler h) {
+ return h.getClass().getName() + '@' + Integer.toHexString(h.hashCode());
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshResponse.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshResponse.java
new file mode 100644
index 000000000000..8d9ce4387d10
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RefreshResponse.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+
+/**
+ * Return a response in the handler method for the user to see.
+ * Useful since you may want to display status to a user even though an
+ * error has not occurred.
+ */
+public class RefreshResponse {
+ private int returnCode = -1;
+ private String message;
+ private String senderName;
+
+ /**
+ * Convenience method to create a response for successful refreshes.
+ * @return void response
+ */
+ public static RefreshResponse successResponse() {
+ return new RefreshResponse(0, "Success");
+ }
+
+ // Most RefreshHandlers will use this
+ public RefreshResponse(int returnCode, String message) {
+ this.returnCode = returnCode;
+ this.message = message;
+ }
+
+ /**
+ * Optionally set the sender of this RefreshResponse.
+ * This helps clarify things when multiple handlers respond.
+ * @param name The name of the sender
+ */
+ public void setSenderName(String name) {
+ senderName = name;
+ }
+ public String getSenderName() { return senderName; }
+
+ public int getReturnCode() { return returnCode; }
+ public void setReturnCode(int rc) { returnCode = rc; }
+
+ public void setMessage(String m) { message = m; }
+ public String getMessage() { return message; }
+
+ @Override
+ public String toString() {
+ String ret = "";
+
+ if (senderName != null) {
+ ret += senderName + ": ";
+ }
+
+ if (message != null) {
+ ret += message;
+ }
+
+ ret += " (exit " + returnCode + ")";
+ return ret;
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RemoteException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RemoteException.java
new file mode 100644
index 000000000000..6a033c4949ec
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RemoteException.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+
+import org.apache.hadoop.ipc_.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.xml.sax.Attributes;
+
+public class RemoteException extends IOException {
+ /** this value should not be defined in RpcHeader.proto so that protobuf will return a null */
+ private static final int UNSPECIFIED_ERROR = -1;
+ /** For java.io.Serializable */
+ private static final long serialVersionUID = 1L;
+ private final int errorCode;
+
+ private final String className;
+
+ /**
+ * @param className wrapped exception, may be null
+ * @param msg may be null
+ */
+ public RemoteException(String className, String msg) {
+ this(className, msg, null);
+ }
+
+ /**
+ * @param className wrapped exception, may be null
+ * @param msg may be null
+ * @param erCode may be null
+ */
+ public RemoteException(String className, String msg, RpcErrorCodeProto erCode) {
+ super(msg);
+ this.className = className;
+ if (erCode != null)
+ errorCode = erCode.getNumber();
+ else
+ errorCode = UNSPECIFIED_ERROR;
+ }
+
+ /**
+ * @return the class name for the wrapped exception; may be null if none was given.
+ */
+ public String getClassName() {
+ return className;
+ }
+
+ /**
+ * @return may be null if the code was newer than our protobuf definitions or none was given.
+ */
+ public RpcErrorCodeProto getErrorCode() {
+ return RpcErrorCodeProto.valueOf(errorCode);
+ }
+
+ /**
+ * If this remote exception wraps up one of the lookupTypes
+ * then return this exception.
+ *
+ * Unwraps any IOException.
+ *
+ * @param lookupTypes the desired exception class. may be null.
+ * @return IOException, which is either the lookupClass exception or this.
+ */
+ public IOException unwrapRemoteException(Class... lookupTypes) {
+ if(lookupTypes == null)
+ return this;
+ for(Class lookupClass : lookupTypes) {
+ if(!lookupClass.getName().equals(getClassName()))
+ continue;
+ try {
+ return instantiateException(lookupClass.asSubclass(IOException.class));
+ } catch(Exception e) {
+ // cannot instantiate lookupClass, just return this
+ return this;
+ }
+ }
+ // wrapped up exception is not in lookupTypes, just return this
+ return this;
+ }
+
+ /**
+ * Instantiate and return the exception wrapped up by this remote exception.
+ *
+ *
This unwraps any Throwable that has a constructor taking
+ * a String as a parameter.
+ * Otherwise it returns this.
+ *
+ * @return Throwable
+ */
+ public IOException unwrapRemoteException() {
+ try {
+ Class realClass = Class.forName(getClassName());
+ return instantiateException(realClass.asSubclass(IOException.class));
+ } catch(Exception e) {
+ // cannot instantiate the original exception, just return this
+ }
+ return this;
+ }
+
+ private IOException instantiateException(Class cls)
+ throws Exception {
+ Constructor cn = cls.getConstructor(String.class);
+ cn.setAccessible(true);
+ IOException ex = cn.newInstance(this.getMessage());
+ ex.initCause(this);
+ return ex;
+ }
+
+ /**
+ * Create RemoteException from attributes.
+ * @param attrs may not be null.
+ * @return RemoteException.
+ */
+ public static RemoteException valueOf(Attributes attrs) {
+ return new RemoteException(attrs.getValue("class"),
+ attrs.getValue("message"));
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getName() + "(" + className + "): " + getMessage();
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ResponseBuffer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ResponseBuffer.java
new file mode 100644
index 000000000000..1f918d82db4b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/ResponseBuffer.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+
+/** generates byte-length framed buffers. */
+public class ResponseBuffer extends DataOutputStream {
+
+ public ResponseBuffer() {
+ this(1024);
+ }
+
+ public ResponseBuffer(int capacity) {
+ super(new FramedBuffer(capacity));
+ }
+
+ // update framing bytes based on bytes written to stream.
+ private FramedBuffer getFramedBuffer() {
+ FramedBuffer buf = (FramedBuffer)out;
+ buf.setSize(written);
+ return buf;
+ }
+
+ public void writeTo(OutputStream out) throws IOException {
+ getFramedBuffer().writeTo(out);
+ }
+
+ byte[] toByteArray() {
+ return getFramedBuffer().toByteArray();
+ }
+
+ int capacity() {
+ return ((FramedBuffer)out).capacity();
+ }
+
+ void setCapacity(int capacity) {
+ ((FramedBuffer)out).setCapacity(capacity);
+ }
+
+ void ensureCapacity(int capacity) {
+ if (((FramedBuffer)out).capacity() < capacity) {
+ ((FramedBuffer)out).setCapacity(capacity);
+ }
+ }
+
+ ResponseBuffer reset() {
+ written = 0;
+ ((FramedBuffer)out).reset();
+ return this;
+ }
+
+ private static class FramedBuffer extends ByteArrayOutputStream {
+ private static final int FRAMING_BYTES = 4;
+ FramedBuffer(int capacity) {
+ super(capacity + FRAMING_BYTES);
+ reset();
+ }
+ @Override
+ public int size() {
+ return count - FRAMING_BYTES;
+ }
+ void setSize(int size) {
+ buf[0] = (byte)((size >>> 24) & 0xFF);
+ buf[1] = (byte)((size >>> 16) & 0xFF);
+ buf[2] = (byte)((size >>> 8) & 0xFF);
+ buf[3] = (byte)((size >>> 0) & 0xFF);
+ }
+ int capacity() {
+ return buf.length - FRAMING_BYTES;
+ }
+ void setCapacity(int capacity) {
+ buf = Arrays.copyOf(buf, capacity + FRAMING_BYTES);
+ }
+ @Override
+ public void reset() {
+ count = FRAMING_BYTES;
+ setSize(0);
+ }
+ };
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RetriableException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RetriableException.java
new file mode 100644
index 000000000000..8c7f50bbca52
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RetriableException.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+
+
+/**
+ * Exception thrown by a server typically to indicate that server is in a state
+ * where request cannot be processed temporarily (such as still starting up).
+ * Client may retry the request. If the service is up, the server may be able to
+ * process a retried request.
+ */
+public class RetriableException extends IOException {
+ private static final long serialVersionUID = 1915561725516487301L;
+
+ public RetriableException(Exception e) {
+ super(e);
+ }
+
+ public RetriableException(String msg) {
+ super(msg);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RetryCache.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RetryCache.java
new file mode 100644
index 000000000000..6467ed56ae61
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RetryCache.java
@@ -0,0 +1,391 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+
+import java.util.Arrays;
+import java.util.UUID;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.ipc_.metrics.RetryCacheMetrics;
+import org.apache.hadoop.util.LightWeightCache;
+import org.apache.hadoop.util.LightWeightGSet;
+import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Maintains a cache of non-idempotent requests that have been successfully
+ * processed by the RPC server implementation, to handle the retries. A request
+ * is uniquely identified by the unique client ID + call ID of the RPC request.
+ * On receiving retried request, an entry will be found in the
+ * {@link RetryCache} and the previous response is sent back to the request.
+ *
+ * To look an implementation using this cache, see HDFS FSNamesystem class.
+ */
+public class RetryCache {
+ public static final Logger LOG = LoggerFactory.getLogger(RetryCache.class);
+ private final RetryCacheMetrics retryCacheMetrics;
+ private static final int MAX_CAPACITY = 16;
+
+ /**
+ * CacheEntry is tracked using unique client ID and callId of the RPC request.
+ */
+ public static class CacheEntry implements LightWeightCache.Entry {
+ /**
+ * Processing state of the requests.
+ */
+ private static byte INPROGRESS = 0;
+ private static byte SUCCESS = 1;
+ private static byte FAILED = 2;
+
+ private byte state = INPROGRESS;
+
+ // Store uuid as two long for better memory utilization
+ private final long clientIdMsb; // Most signficant bytes
+ private final long clientIdLsb; // Least significant bytes
+
+ private final int callId;
+ private final long expirationTime;
+ private LightWeightGSet.LinkedElement next;
+
+ CacheEntry(byte[] clientId, int callId, long expirationTime) {
+ // ClientId must be a UUID - that is 16 octets.
+ Preconditions.checkArgument(clientId.length == ClientId.BYTE_LENGTH,
+ "Invalid clientId - length is " + clientId.length
+ + " expected length " + ClientId.BYTE_LENGTH);
+ // Convert UUID bytes to two longs
+ clientIdMsb = ClientId.getMsb(clientId);
+ clientIdLsb = ClientId.getLsb(clientId);
+ this.callId = callId;
+ this.expirationTime = expirationTime;
+ }
+
+ CacheEntry(byte[] clientId, int callId, long expirationTime,
+ boolean success) {
+ this(clientId, callId, expirationTime);
+ this.state = success ? SUCCESS : FAILED;
+ }
+
+ private static int hashCode(long value) {
+ return (int)(value ^ (value >>> 32));
+ }
+
+ @Override
+ public int hashCode() {
+ return (hashCode(clientIdMsb) * 31 + hashCode(clientIdLsb)) * 31 + callId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof CacheEntry)) {
+ return false;
+ }
+ CacheEntry other = (CacheEntry) obj;
+ return callId == other.callId && clientIdMsb == other.clientIdMsb
+ && clientIdLsb == other.clientIdLsb;
+ }
+
+ @Override
+ public void setNext(LinkedElement next) {
+ this.next = next;
+ }
+
+ @Override
+ public LinkedElement getNext() {
+ return next;
+ }
+
+ synchronized void completed(boolean success) {
+ state = success ? SUCCESS : FAILED;
+ this.notifyAll();
+ }
+
+ public synchronized boolean isSuccess() {
+ return state == SUCCESS;
+ }
+
+ @Override
+ public void setExpirationTime(long timeNano) {
+ // expiration time does not change
+ }
+
+ @Override
+ public long getExpirationTime() {
+ return expirationTime;
+ }
+
+ @Override
+ public String toString() {
+ return (new UUID(this.clientIdMsb, this.clientIdLsb)).toString() + ":"
+ + this.callId + ":" + this.state;
+ }
+ }
+
+ /**
+ * CacheEntry with payload that tracks the previous response or parts of
+ * previous response to be used for generating response for retried requests.
+ */
+ public static class CacheEntryWithPayload extends CacheEntry {
+ private Object payload;
+
+ CacheEntryWithPayload(byte[] clientId, int callId, Object payload,
+ long expirationTime) {
+ super(clientId, callId, expirationTime);
+ this.payload = payload;
+ }
+
+ CacheEntryWithPayload(byte[] clientId, int callId, Object payload,
+ long expirationTime, boolean success) {
+ super(clientId, callId, expirationTime, success);
+ this.payload = payload;
+ }
+
+ /** Override equals to avoid findbugs warnings */
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ /** Override hashcode to avoid findbugs warnings */
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ public Object getPayload() {
+ return payload;
+ }
+ }
+
+ private final LightWeightGSet set;
+ private final long expirationTime;
+ private String cacheName;
+
+ private final ReentrantLock lock = new ReentrantLock();
+
+ /**
+ * Constructor
+ * @param cacheName name to identify the cache by
+ * @param percentage percentage of total java heap space used by this cache
+ * @param expirationTime time for an entry to expire in nanoseconds
+ */
+ public RetryCache(String cacheName, double percentage, long expirationTime) {
+ int capacity = LightWeightGSet.computeCapacity(percentage, cacheName);
+ capacity = capacity > MAX_CAPACITY ? capacity : MAX_CAPACITY;
+ this.set = new LightWeightCache(capacity, capacity,
+ expirationTime, 0);
+ this.expirationTime = expirationTime;
+ this.cacheName = cacheName;
+ this.retryCacheMetrics = RetryCacheMetrics.create(this);
+ }
+
+ private static boolean skipRetryCache() {
+ // Do not track non RPC invocation or RPC requests with
+ // invalid callId or clientId in retry cache
+ return !Server.isRpcInvocation() || Server.getCallId() < 0
+ || Arrays.equals(Server.getClientId(), RpcConstants.DUMMY_CLIENT_ID);
+ }
+
+ public void lock() {
+ this.lock.lock();
+ }
+
+ public void unlock() {
+ this.lock.unlock();
+ }
+
+ private void incrCacheClearedCounter() {
+ retryCacheMetrics.incrCacheCleared();
+ }
+
+ public LightWeightGSet getCacheSet() {
+ return set;
+ }
+
+ public RetryCacheMetrics getMetricsForTests() {
+ return retryCacheMetrics;
+ }
+
+ /**
+ * @return This method returns cache name for metrics.
+ */
+ public String getCacheName() {
+ return cacheName;
+ }
+
+ /**
+ * This method handles the following conditions:
+ *
+ * If retry is not to be processed, return null
+ * If there is no cache entry, add a new entry {@code newEntry} and return
+ * it.
+ * If there is an existing entry, wait for its completion. If the
+ * completion state is {@link CacheEntry#FAILED}, the expectation is that the
+ * thread that waited for completion, retries the request. the
+ * {@link CacheEntry} state is set to {@link CacheEntry#INPROGRESS} again.
+ * If the completion state is {@link CacheEntry#SUCCESS}, the entry is
+ * returned so that the thread that waits for it can can return previous
+ * response.
+ *
+ *
+ * @return {@link CacheEntry}.
+ */
+ private CacheEntry waitForCompletion(CacheEntry newEntry) {
+ CacheEntry mapEntry = null;
+ lock.lock();
+ try {
+ mapEntry = set.get(newEntry);
+ // If an entry in the cache does not exist, add a new one
+ if (mapEntry == null) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Adding Rpc request clientId "
+ + newEntry.clientIdMsb + newEntry.clientIdLsb + " callId "
+ + newEntry.callId + " to retryCache");
+ }
+ set.put(newEntry);
+ retryCacheMetrics.incrCacheUpdated();
+ return newEntry;
+ } else {
+ retryCacheMetrics.incrCacheHit();
+ }
+ } finally {
+ lock.unlock();
+ }
+ // Entry already exists in cache. Wait for completion and return its state
+ Preconditions.checkNotNull(mapEntry,
+ "Entry from the cache should not be null");
+ // Wait for in progress request to complete
+ synchronized (mapEntry) {
+ while (mapEntry.state == CacheEntry.INPROGRESS) {
+ try {
+ mapEntry.wait();
+ } catch (InterruptedException ie) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ }
+ }
+ // Previous request has failed, the expectation is is that it will be
+ // retried again.
+ if (mapEntry.state != CacheEntry.SUCCESS) {
+ mapEntry.state = CacheEntry.INPROGRESS;
+ }
+ }
+ return mapEntry;
+ }
+
+ /**
+ * Add a new cache entry into the retry cache. The cache entry consists of
+ * clientId and callId extracted from editlog.
+ *
+ * @param clientId input clientId.
+ * @param callId input callId.
+ */
+ public void addCacheEntry(byte[] clientId, int callId) {
+ CacheEntry newEntry = new CacheEntry(clientId, callId, System.nanoTime()
+ + expirationTime, true);
+ lock.lock();
+ try {
+ set.put(newEntry);
+ } finally {
+ lock.unlock();
+ }
+ retryCacheMetrics.incrCacheUpdated();
+ }
+
+ public void addCacheEntryWithPayload(byte[] clientId, int callId,
+ Object payload) {
+ // since the entry is loaded from editlog, we can assume it succeeded.
+ CacheEntry newEntry = new CacheEntryWithPayload(clientId, callId, payload,
+ System.nanoTime() + expirationTime, true);
+ lock.lock();
+ try {
+ set.put(newEntry);
+ } finally {
+ lock.unlock();
+ }
+ retryCacheMetrics.incrCacheUpdated();
+ }
+
+ private static CacheEntry newEntry(long expirationTime) {
+ return new CacheEntry(Server.getClientId(), Server.getCallId(),
+ System.nanoTime() + expirationTime);
+ }
+
+ private static CacheEntryWithPayload newEntry(Object payload,
+ long expirationTime) {
+ return new CacheEntryWithPayload(Server.getClientId(), Server.getCallId(),
+ payload, System.nanoTime() + expirationTime);
+ }
+
+ /**
+ * Static method that provides null check for retryCache.
+ * @param cache input Cache.
+ * @return CacheEntry.
+ */
+ public static CacheEntry waitForCompletion(RetryCache cache) {
+ if (skipRetryCache()) {
+ return null;
+ }
+ return cache != null ? cache
+ .waitForCompletion(newEntry(cache.expirationTime)) : null;
+ }
+
+ /**
+ * Static method that provides null check for retryCache.
+ * @param cache input cache.
+ * @param payload input payload.
+ * @return CacheEntryWithPayload.
+ */
+ public static CacheEntryWithPayload waitForCompletion(RetryCache cache,
+ Object payload) {
+ if (skipRetryCache()) {
+ return null;
+ }
+ return (CacheEntryWithPayload) (cache != null ? cache
+ .waitForCompletion(newEntry(payload, cache.expirationTime)) : null);
+ }
+
+ public static void setState(CacheEntry e, boolean success) {
+ if (e == null) {
+ return;
+ }
+ e.completed(success);
+ }
+
+ public static void setState(CacheEntryWithPayload e, boolean success,
+ Object payload) {
+ if (e == null) {
+ return;
+ }
+ e.payload = payload;
+ e.completed(success);
+ }
+
+ public static void clear(RetryCache cache) {
+ if (cache != null) {
+ cache.set.clear();
+ cache.incrCacheClearedCounter();
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcClientException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcClientException.java
new file mode 100644
index 000000000000..65e02494b941
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcClientException.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+/**
+ * Indicates an exception in the RPC client
+ */
+public class RpcClientException extends RpcException {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructs exception with the specified detail message.
+ *
+ * @param messages detailed message.
+ */
+ RpcClientException(final String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs exception with the specified detail message and cause.
+ *
+ * @param message message.
+ * @param cause that cause this exception
+ * @param cause the cause (can be retried by the {@link #getCause()} method).
+ * (A null value is permitted, and indicates that the cause
+ * is nonexistent or unknown.)
+ */
+ RpcClientException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcClientUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcClientUtil.java
new file mode 100644
index 000000000000..1683e2cd681c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcClientUtil.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
+import org.apache.hadoop.ipc_.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
+import org.apache.hadoop.net.NetUtils;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+/**
+ * This class maintains a cache of protocol versions and corresponding protocol
+ * signatures, keyed by server address, protocol and rpc kind.
+ * The cache is lazily populated.
+ */
+public class RpcClientUtil {
+ private static RpcController NULL_CONTROLLER = null;
+ private static final int PRIME = 16777619;
+
+ private static class ProtoSigCacheKey {
+ private InetSocketAddress serverAddress;
+ private String protocol;
+ private String rpcKind;
+
+ ProtoSigCacheKey(InetSocketAddress addr, String p, String rk) {
+ this.serverAddress = addr;
+ this.protocol = p;
+ this.rpcKind = rk;
+ }
+
+ @Override //Object
+ public int hashCode() {
+ int result = 1;
+ result = PRIME * result
+ + ((serverAddress == null) ? 0 : serverAddress.hashCode());
+ result = PRIME * result + ((protocol == null) ? 0 : protocol.hashCode());
+ result = PRIME * result + ((rpcKind == null) ? 0 : rpcKind.hashCode());
+ return result;
+ }
+
+ @Override //Object
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (other instanceof ProtoSigCacheKey) {
+ ProtoSigCacheKey otherKey = (ProtoSigCacheKey) other;
+ return (serverAddress.equals(otherKey.serverAddress) &&
+ protocol.equals(otherKey.protocol) &&
+ rpcKind.equals(otherKey.rpcKind));
+ }
+ return false;
+ }
+ }
+
+ private static ConcurrentHashMap>
+ signatureMap = new ConcurrentHashMap>();
+
+ private static void putVersionSignatureMap(InetSocketAddress addr,
+ String protocol, String rpcKind, Map map) {
+ signatureMap.put(new ProtoSigCacheKey(addr, protocol, rpcKind), map);
+ }
+
+ private static Map getVersionSignatureMap(
+ InetSocketAddress addr, String protocol, String rpcKind) {
+ return signatureMap.get(new ProtoSigCacheKey(addr, protocol, rpcKind));
+ }
+
+ /**
+ * Returns whether the given method is supported or not.
+ * The protocol signatures are fetched and cached. The connection id for the
+ * proxy provided is re-used.
+ * @param rpcProxy Proxy which provides an existing connection id.
+ * @param protocol Protocol for which the method check is required.
+ * @param rpcKind The RpcKind for which the method check is required.
+ * @param version The version at the client.
+ * @param methodName Name of the method.
+ * @return true if the method is supported, false otherwise.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public static boolean isMethodSupported(Object rpcProxy, Class protocol,
+ RPC.RpcKind rpcKind, long version, String methodName) throws IOException {
+ InetSocketAddress serverAddress = RPC.getServerAddress(rpcProxy);
+ Map versionMap = getVersionSignatureMap(
+ serverAddress, protocol.getName(), rpcKind.toString());
+
+ if (versionMap == null) {
+ Configuration conf = new Configuration();
+ RPC.setProtocolEngine(conf, ProtocolMetaInfoPB.class,
+ ProtobufRpcEngine.class);
+ ProtocolMetaInfoPB protocolInfoProxy = getProtocolMetaInfoProxy(rpcProxy,
+ conf);
+ GetProtocolSignatureRequestProto.Builder builder =
+ GetProtocolSignatureRequestProto.newBuilder();
+ builder.setProtocol(protocol.getName());
+ builder.setRpcKind(rpcKind.toString());
+ GetProtocolSignatureResponseProto resp;
+ try {
+ resp = protocolInfoProxy.getProtocolSignature(NULL_CONTROLLER,
+ builder.build());
+ } catch (ServiceException se) {
+ throw ProtobufHelper.getRemoteException(se);
+ }
+ versionMap = convertProtocolSignatureProtos(resp
+ .getProtocolSignatureList());
+ putVersionSignatureMap(serverAddress, protocol.getName(),
+ rpcKind.toString(), versionMap);
+ }
+ // Assuming unique method names.
+ Method desiredMethod;
+ Method[] allMethods = protocol.getMethods();
+ desiredMethod = null;
+ for (Method m : allMethods) {
+ if (m.getName().equals(methodName)) {
+ desiredMethod = m;
+ break;
+ }
+ }
+ if (desiredMethod == null) {
+ return false;
+ }
+ int methodHash = ProtocolSignature.getFingerprint(desiredMethod);
+ return methodExists(methodHash, version, versionMap);
+ }
+
+ private static Map
+ convertProtocolSignatureProtos(List protoList) {
+ Map map = new TreeMap();
+ for (ProtocolSignatureProto p : protoList) {
+ int [] methods = new int[p.getMethodsList().size()];
+ int index=0;
+ for (int m : p.getMethodsList()) {
+ methods[index++] = m;
+ }
+ map.put(p.getVersion(), new ProtocolSignature(p.getVersion(), methods));
+ }
+ return map;
+ }
+
+ private static boolean methodExists(int methodHash, long version,
+ Map versionMap) {
+ ProtocolSignature sig = versionMap.get(version);
+ if (sig != null) {
+ for (int m : sig.getMethods()) {
+ if (m == methodHash) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ // The proxy returned re-uses the underlying connection. This is a special
+ // mechanism for ProtocolMetaInfoPB.
+ // Don't do this for any other protocol, it might cause a security hole.
+ private static ProtocolMetaInfoPB getProtocolMetaInfoProxy(Object proxy,
+ Configuration conf) throws IOException {
+ RpcInvocationHandler inv = (RpcInvocationHandler) Proxy
+ .getInvocationHandler(proxy);
+ return RPC
+ .getProtocolEngine(ProtocolMetaInfoPB.class, conf)
+ .getProtocolMetaInfoProxy(inv.getConnectionId(), conf,
+ NetUtils.getDefaultSocketFactory(conf)).getProxy();
+ }
+
+ /**
+ * Convert an RPC method to a string.
+ * The format we want is 'MethodOuterClassShortName#methodName'.
+ *
+ * For example, if the method is:
+ * org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.
+ * ClientNamenodeProtocol.BlockingInterface.getServerDefaults
+ *
+ * the format we want is:
+ * ClientNamenodeProtocol#getServerDefaults
+ * @param method input method.
+ * @return methodToTraceString.
+ */
+ public static String methodToTraceString(Method method) {
+ Class clazz = method.getDeclaringClass();
+ while (true) {
+ Class next = clazz.getEnclosingClass();
+ if (next == null || next.getEnclosingClass() == null) break;
+ clazz = next;
+ }
+ return clazz.getSimpleName() + "#" + method.getName();
+ }
+
+ /**
+ * Convert an RPC class method to a string.
+ * The format we want is
+ * 'SecondOutermostClassShortName#OutermostClassShortName'.
+ *
+ * For example, if the full class name is:
+ * org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations
+ *
+ * the format we want is:
+ * ClientProtocol#getBlockLocations
+ * @param fullName input fullName.
+ * @return toTraceName.
+ */
+ public static String toTraceName(String fullName) {
+ int lastPeriod = fullName.lastIndexOf('.');
+ if (lastPeriod < 0) {
+ return fullName;
+ }
+ int secondLastPeriod = fullName.lastIndexOf('.', lastPeriod - 1);
+ if (secondLastPeriod < 0) {
+ return fullName;
+ }
+ return fullName.substring(secondLastPeriod + 1, lastPeriod) + "#" +
+ fullName.substring(lastPeriod + 1);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcConstants.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcConstants.java
new file mode 100644
index 000000000000..05b7ac5561b7
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcConstants.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc_;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+
+public class RpcConstants {
+ private RpcConstants() {
+ // Hidden Constructor
+ }
+
+ public static final int AUTHORIZATION_FAILED_CALL_ID = -1;
+ public static final int INVALID_CALL_ID = -2;
+ public static final int CONNECTION_CONTEXT_CALL_ID = -3;
+ public static final int PING_CALL_ID = -4;
+
+ public static final byte[] DUMMY_CLIENT_ID = new byte[0];
+
+
+ public static final int INVALID_RETRY_COUNT = -1;
+
+ /**
+ * The Rpc-connection header is as follows
+ * +----------------------------------+
+ * | "hrpc" 4 bytes |
+ * +----------------------------------+
+ * | Version (1 byte) |
+ * +----------------------------------+
+ * | Service Class (1 byte) |
+ * +----------------------------------+
+ * | AuthProtocol (1 byte) |
+ * +----------------------------------+
+ */
+
+ /**
+ * The first four bytes of Hadoop RPC connections
+ */
+ public static final ByteBuffer HEADER =
+ ByteBuffer.wrap("hrpc".getBytes(StandardCharsets.UTF_8));
+ public static final int HEADER_LEN_AFTER_HRPC_PART = 3; // 3 bytes that follow
+
+ // 1 : Introduce ping and server does not throw away RPCs
+ // 3 : Introduce the protocol into the RPC connection header
+ // 4 : Introduced SASL security layer
+ // 5 : Introduced use of {@link ArrayPrimitiveWritable$Internal}
+ // in ObjectWritable to efficiently transmit arrays of primitives
+ // 6 : Made RPC Request header explicit
+ // 7 : Changed Ipc Connection Header to use Protocol buffers
+ // 8 : SASL server always sends a final response
+ // 9 : Changes to protocol for HADOOP-8990
+ public static final byte CURRENT_VERSION = 9;
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcEngine.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcEngine.java
new file mode 100644
index 000000000000..473bf78ef042
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/RpcEngine.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc_;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc_.Client.ConnectionId;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+/** An RPC implementation. */
+public interface RpcEngine {
+
+ /**
+ * Construct a client-side proxy object.
+ *
+ * @param Generics Type T.
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param addr input addr.
+ * @param ticket input ticket.
+ * @param conf input Configuration.
+ * @param factory input factory.
+ * @param rpcTimeout input rpcTimeout.
+ * @param connectionRetryPolicy input connectionRetryPolicy.
+ * @throws IOException raised on errors performing I/O.
+ * @return ProtocolProxy.
+ */
+ ProtocolProxy getProxy(Class protocol,
+ long clientVersion, InetSocketAddress addr,
+ UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout,
+ RetryPolicy connectionRetryPolicy) throws IOException;
+
+ /**
+ * Construct a client-side proxy object with a ConnectionId.
+ *
+ * @param Generics Type T.
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param connId input ConnectionId.
+ * @param conf input Configuration.
+ * @param factory input factory.
+ * @throws IOException raised on errors performing I/O.
+ * @return ProtocolProxy.
+ */
+ ProtocolProxy getProxy(Class protocol, long clientVersion,
+ Client.ConnectionId connId, Configuration conf, SocketFactory factory)
+ throws IOException;
+
+ /**
+ * Construct a client-side proxy object.
+ *
+ * @param Generics Type T.
+ * @param protocol input protocol.
+ * @param clientVersion input clientVersion.
+ * @param addr input addr.
+ * @param ticket input tocket.
+ * @param conf input Configuration.
+ * @param factory input factory.
+ * @param rpcTimeout input rpcTimeout.
+ * @param connectionRetryPolicy input connectionRetryPolicy.
+ * @param fallbackToSimpleAuth input fallbackToSimpleAuth.
+ * @param alignmentContext input alignmentContext.
+ * @throws IOException raised on errors performing I/O.
+ * @return ProtocolProxy.
+ */
+ ProtocolProxy