Search code examples
javahadoophdfshadoop-yarnkerberos

Kerberos and yarn nodemanager fail to connecting resourcemanager. Failed to specify server's Kerberos principal name

I`m trying to run my cluster with Kerberos. Before hdfs, yarn and spark worked correctly. After setting up kerberos, I can only run hdfs because yarn is going to crush after 15 minutes with an error. I have tried different configurations with no result. Master node do not have any logs about slave node. Nodemanager run just for 15 minutes but do not show on yarn master list.

I do not understand why you can use kinit or hdfs run with no problem, but yarn seems to not connect the resource manager.

Log:

[...]
2020-12-02 22:23:07,532 INFO org.eclipse.jetty.server.AbstractConnector: Started ServerConnector@333cb916{HTTP/1.1,[http/1.1]}{0.0.0.0:8042}
2020-12-02 22:23:07,532 INFO org.eclipse.jetty.server.Server: Started @3804ms
2020-12-02 22:23:07,532 INFO org.apache.hadoop.yarn.webapp.WebApps: Web app node started at 8042
2020-12-02 22:23:07,534 INFO org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl: Node ID assigned is : node1:46673
2020-12-02 22:23:07,537 INFO org.apache.hadoop.util.JvmPauseMonitor: Starting JVM pause monitor
2020-12-02 22:23:07,541 INFO org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider: Connecting to ResourceManager at master.ar.com/192.168.46.100:8031
2020-12-02 22:23:07,567 INFO org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl: Sending out 0 NM container statuses: []
2020-12-02 22:23:07,576 INFO org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl: Registering with RM using containers :[]
2020-12-02 22:33:05,974 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService: Cache Size Before Clean: 0, Total Deleted: 0, Public Deleted: 0, Private Deleted: 0
2020-12-02 22:38:07,867 ERROR org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl: Unexpected error starting NodeStatusUpdater
java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at java.base/jdk.internal.reflect.GeneratedConstructorAccessor23.newInstance(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
        at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:837)
        at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:812)
        at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566)
        at org.apache.hadoop.ipc.Client.call(Client.java:1508)
        at org.apache.hadoop.ipc.Client.call(Client.java:1405)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:234)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:119)
        at com.sun.proxy.$Proxy76.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:74)
        at jdk.internal.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
        at com.sun.proxy.$Proxy77.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:416)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:274)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:122)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:963)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:1042)
Caused by: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:884)
        at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:413)
        at org.apache.hadoop.ipc.Client.getConnection(Client.java:1636)
        at org.apache.hadoop.ipc.Client.call(Client.java:1452)
        ... 21 more
Caused by: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.security.SaslRpcClient.getServerPrincipal(SaslRpcClient.java:333)
        at org.apache.hadoop.security.SaslRpcClient.createSaslClient(SaslRpcClient.java:240)
        at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:166)
        at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:392)
        at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:622)
        at org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:413)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:822)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:818)
        at java.base/java.security.AccessController.doPrivileged(Native Method)
        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1845)
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:818)
        ... 24 more
2020-12-02 22:38:07,870 INFO org.apache.hadoop.service.AbstractService: Service org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl failed in state STARTED
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:280)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:122)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:963)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:1042)
Caused by: java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at java.base/jdk.internal.reflect.GeneratedConstructorAccessor23.newInstance(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
        at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:837)
        at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:812)
        at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566)
        at org.apache.hadoop.ipc.Client.call(Client.java:1508)
        at org.apache.hadoop.ipc.Client.call(Client.java:1405)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:234)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:119)
        at com.sun.proxy.$Proxy76.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:74)
        at jdk.internal.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
        at com.sun.proxy.$Proxy77.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:416)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:274)
        ... 5 more
Caused by: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:884)
        at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:413)
        at org.apache.hadoop.ipc.Client.getConnection(Client.java:1636)
        at org.apache.hadoop.ipc.Client.call(Client.java:1452)
        ... 21 more
Caused by: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.security.SaslRpcClient.getServerPrincipal(SaslRpcClient.java:333)
        at org.apache.hadoop.security.SaslRpcClient.createSaslClient(SaslRpcClient.java:240)
        at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:166)
        at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:392)
        at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:622)
        at org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:413)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:822)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:818)
        at java.base/java.security.AccessController.doPrivileged(Native Method)
        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1845)
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:818)
        ... 24 more
2020-12-02 22:38:07,871 INFO org.apache.hadoop.service.AbstractService: Service NodeManager failed in state STARTED
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:280)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:122)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:963)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:1042)
Caused by: java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at java.base/jdk.internal.reflect.GeneratedConstructorAccessor23.newInstance(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
        at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:837)
        at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:812)
        at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566)
        at org.apache.hadoop.ipc.Client.call(Client.java:1508)
        at org.apache.hadoop.ipc.Client.call(Client.java:1405)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:234)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:119)
        at com.sun.proxy.$Proxy76.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:74)
        at jdk.internal.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
        at com.sun.proxy.$Proxy77.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:416)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:274)
        ... 5 more
Caused by: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:884)
        at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:413)
        at org.apache.hadoop.ipc.Client.getConnection(Client.java:1636)
        at org.apache.hadoop.ipc.Client.call(Client.java:1452)
        ... 21 more
Caused by: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.security.SaslRpcClient.getServerPrincipal(SaslRpcClient.java:333)
        at org.apache.hadoop.security.SaslRpcClient.createSaslClient(SaslRpcClient.java:240)
        at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:166)
        at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:392)
        at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:622)
        at org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:413)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:822)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:818)
        at java.base/java.security.AccessController.doPrivileged(Native Method)
        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1845)
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:818)
        ... 24 more
2020-12-02 22:38:07,877 INFO org.eclipse.jetty.server.handler.ContextHandler: Stopped o.e.j.w.WebAppContext@6fa02932{node,/,null,UNAVAILABLE}{jar:file:/home/hadoop/hadoop-3.3.0/share/hadoop/yarn/hadoop-yarn-common-3.3.0.jar!/webapps/node}
2020-12-02 22:38:07,880 INFO org.eclipse.jetty.server.AbstractConnector: Stopped ServerConnector@333cb916{HTTP/1.1,[http/1.1]}{0.0.0.0:8042}
2020-12-02 22:38:07,880 INFO org.eclipse.jetty.server.session: node0 Stopped scavenging
2020-12-02 22:38:07,881 INFO org.eclipse.jetty.server.handler.ContextHandler: Stopped o.e.j.s.ServletContextHandler@aac3f4e{static,/static,jar:file:/home/hadoop/hadoop-3.3.0/share/hadoop/yarn/hadoop-yarn-common-3.3.0.jar!/webapps/static,UNAVAILABLE}
2020-12-02 22:38:07,881 INFO org.eclipse.jetty.server.handler.ContextHandler: Stopped o.e.j.s.ServletContextHandler@2b4786dd{logs,/logs,file:///home/hadoop/hadoop-3.3.0/logs/,UNAVAILABLE}
2020-12-02 22:38:07,884 INFO org.apache.hadoop.ipc.Server: Stopping server on 46673
2020-12-02 22:38:07,895 INFO org.apache.hadoop.ipc.Server: Stopping IPC Server listener on 0
2020-12-02 22:38:07,895 INFO org.apache.hadoop.ipc.Server: Stopping IPC Server Responder
2020-12-02 22:38:07,896 WARN org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl: org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl is interrupted. Exiting.
2020-12-02 22:38:07,911 INFO org.apache.hadoop.ipc.Server: Stopping server on 8040
2020-12-02 22:38:07,912 INFO org.apache.hadoop.ipc.Server: Stopping IPC Server listener on 8040
2020-12-02 22:38:07,915 INFO org.apache.hadoop.ipc.Server: Stopping IPC Server Responder
2020-12-02 22:38:07,916 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService: Public cache exiting
2020-12-02 22:38:07,917 WARN org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl: org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl is interrupted. Exiting.
2020-12-02 22:38:07,920 INFO org.apache.hadoop.metrics2.impl.MetricsSystemImpl: Stopping NodeManager metrics system...
2020-12-02 22:38:07,921 INFO org.apache.hadoop.metrics2.impl.MetricsSystemImpl: NodeManager metrics system stopped.
2020-12-02 22:38:07,921 INFO org.apache.hadoop.metrics2.impl.MetricsSystemImpl: NodeManager metrics system shutdown complete.
2020-12-02 22:38:07,921 ERROR org.apache.hadoop.yarn.server.nodemanager.NodeManager: Error starting NodeManager
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:280)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:122)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:963)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:1042)
Caused by: java.io.IOException: DestHost:destPort master.ar.com:8031 , LocalHost:localPort node1.ar.com/192.168.46.101:0. Failed on local exception: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at java.base/jdk.internal.reflect.GeneratedConstructorAccessor23.newInstance(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
        at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:837)
        at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:812)
        at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566)
        at org.apache.hadoop.ipc.Client.call(Client.java:1508)
        at org.apache.hadoop.ipc.Client.call(Client.java:1405)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:234)
        at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:119)
        at com.sun.proxy.$Proxy76.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:74)
        at jdk.internal.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
        at com.sun.proxy.$Proxy77.registerNodeManager(Unknown Source)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:416)
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:274)
        ... 5 more
Caused by: java.io.IOException: Couldn't set up IO streams: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:884)
        at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:413)
        at org.apache.hadoop.ipc.Client.getConnection(Client.java:1636)
        at org.apache.hadoop.ipc.Client.call(Client.java:1452)
        ... 21 more
Caused by: java.lang.IllegalArgumentException: Failed to specify server's Kerberos principal name
        at org.apache.hadoop.security.SaslRpcClient.getServerPrincipal(SaslRpcClient.java:333)
        at org.apache.hadoop.security.SaslRpcClient.createSaslClient(SaslRpcClient.java:240)
        at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:166)
        at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:392)
        at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:622)
        at org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:413)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:822)
        at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:818)
        at java.base/java.security.AccessController.doPrivileged(Native Method)
        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1845)
        at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:818)
        ... 24 more
2020-12-02 22:38:07,926 INFO org.apache.hadoop.yarn.server.nodemanager.NodeManager: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NodeManager at node1.ar.com/192.168.46.101
************************************************************/

core-site.xml

<configuration>
<property>
            <name>fs.default.name</name>
            <value>hdfs://master.ar.com:9000</value>
        </property>

<property>
 <name>hadoop.security.authentication</name>
 <value>kerberos</value>
</property>

<property>
 <name>hadoop.security.authorization</name>
 <value>true</value>
</property>

</configuration>

hdfs-site.xml

<configuration>

    <property>
        <name>dfs.name.dir</name>
        <value>/home/hadoop/hdfs/namenode</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/home/hadoop/hdfs/datanode</value>
    </property>


<property>
 <name>dfs.block.access.token.enable</name>
 <value>true</value>
</property>

<property>
 <name>dfs.client.read.shortcircuit</name>
 <value>true</value>
</property>

<property>
 <name>dfs.datanode.data.dir.perm</name>
 <value>700</value>
</property>

<property>
 <name>dfs.datanode.address</name>
 <value>0.0.0.0:1004</value>
</property>

<property>
 <name>dfs.datanode.http.address</name>
 <value>0.0.0.0:1006</value>
</property>

<property>
 <name>dfs.datanode.kerberos.principal</name>
 <value>hdfs/[email protected]</value>
</property>

<property>
 <name>dfs.datanode.kerberos.http.principal</name>
 <value>HTTP/[email protected]</value>
</property>

<property>
 <name>dfs.datanode.keytab.file</name>
 <value>/etc/security/keytab/hdfs.service.keytab</value>
</property>

</configuration>

yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master.ar.com</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>


<property>
 <name>yarn.nodemanager.principal</name>
 <value>yarn/[email protected]</value>
</property>

<property>
 <name>yarn.nodemanager.keytab</name>
 <value>/etc/security/keytab/yarn.service.keytab</value>
</property>

<property>
 <name>yarn.nodemanager.container-executor.class</name>
 <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
</property>

<property>
 <name>yarn.nodemanager.linux-container-executor.group</name>
 <value>yarn</value>
</property>

</configuration>
Solution

  • Looks like you are missing yarn.resourcemanager.principal. Try adding below configuration on NodeManager's yarn-site.xml.

    <property>
  <name>yarn.resourcemanager.principal</name>
  <value>yarn/[email protected]</value>
</property>