001/** 002* Licensed to the Apache Software Foundation (ASF) under one 003* or more contributor license agreements. See the NOTICE file 004* distributed with this work for additional information 005* regarding copyright ownership. The ASF licenses this file 006* to you under the Apache License, Version 2.0 (the 007* "License"); you may not use this file except in compliance 008* with the License. You may obtain a copy of the License at 009* 010* http://www.apache.org/licenses/LICENSE-2.0 011* 012* Unless required by applicable law or agreed to in writing, software 013* distributed under the License is distributed on an "AS IS" BASIS, 014* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015* See the License for the specific language governing permissions and 016* limitations under the License. 017*/ 018 019package org.apache.hadoop.yarn.conf; 020 021import java.net.InetSocketAddress; 022import java.util.Arrays; 023import java.util.Collections; 024import java.util.List; 025 026import org.apache.hadoop.HadoopIllegalArgumentException; 027import org.apache.hadoop.classification.InterfaceAudience.Private; 028import org.apache.hadoop.classification.InterfaceAudience.Public; 029import org.apache.hadoop.classification.InterfaceStability.Evolving; 030import org.apache.hadoop.classification.InterfaceStability.Unstable; 031import org.apache.hadoop.conf.Configuration; 032import org.apache.hadoop.http.HttpConfig; 033import org.apache.hadoop.net.NetUtils; 034import org.apache.hadoop.security.authorize.ProxyUsers; 035import org.apache.hadoop.util.StringUtils; 036import org.apache.hadoop.yarn.api.ApplicationConstants; 037 038@Public 039@Evolving 040public class YarnConfiguration extends Configuration { 041 042 @Private 043 public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; 044 045 @Private 046 public static final String HADOOP_POLICY_CONFIGURATION_FILE = 047 "hadoop-policy.xml"; 048 049 @Private 050 public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml"; 051 052 private static final String YARN_DEFAULT_CONFIGURATION_FILE = 053 "yarn-default.xml"; 054 055 @Private 056 public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; 057 058 @Private 059 public static final List<String> RM_CONFIGURATION_FILES = 060 Collections.unmodifiableList(Arrays.asList( 061 CS_CONFIGURATION_FILE, 062 HADOOP_POLICY_CONFIGURATION_FILE, 063 YARN_SITE_CONFIGURATION_FILE, 064 CORE_SITE_CONFIGURATION_FILE)); 065 066 @Evolving 067 public static final int APPLICATION_MAX_TAGS = 10; 068 069 @Evolving 070 public static final int APPLICATION_MAX_TAG_LENGTH = 100; 071 072 static { 073 addDeprecatedKeys(); 074 Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE); 075 Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE); 076 } 077 078 private static void addDeprecatedKeys() { 079 Configuration.addDeprecations(new DeprecationDelta[] { 080 new DeprecationDelta("yarn.client.max-nodemanagers-proxies", 081 NM_CLIENT_MAX_NM_PROXIES) 082 }); 083 } 084 085 //Configurations 086 087 public static final String YARN_PREFIX = "yarn."; 088 089 /** Delay before deleting resource to ease debugging of NM issues */ 090 public static final String DEBUG_NM_DELETE_DELAY_SEC = 091 YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; 092 093 //////////////////////////////// 094 // IPC Configs 095 //////////////////////////////// 096 public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; 097 098 /** Factory to create client IPC classes.*/ 099 public static final String IPC_CLIENT_FACTORY_CLASS = 100 IPC_PREFIX + "client.factory.class"; 101 public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = 102 "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl"; 103 104 /** Factory to create server IPC classes.*/ 105 public static final String IPC_SERVER_FACTORY_CLASS = 106 IPC_PREFIX + "server.factory.class"; 107 public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = 108 "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl"; 109 110 /** Factory to create serializeable records.*/ 111 public static final String IPC_RECORD_FACTORY_CLASS = 112 IPC_PREFIX + "record.factory.class"; 113 public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = 114 "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl"; 115 116 /** RPC class implementation*/ 117 public static final String IPC_RPC_IMPL = 118 IPC_PREFIX + "rpc.class"; 119 public static final String DEFAULT_IPC_RPC_IMPL = 120 "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; 121 122 //////////////////////////////// 123 // Resource Manager Configs 124 //////////////////////////////// 125 public static final String RM_PREFIX = "yarn.resourcemanager."; 126 127 public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id"; 128 129 public static final String RM_HOSTNAME = RM_PREFIX + "hostname"; 130 131 /** The address of the applications manager interface in the RM.*/ 132 public static final String RM_ADDRESS = 133 RM_PREFIX + "address"; 134 public static final int DEFAULT_RM_PORT = 8032; 135 public static final String DEFAULT_RM_ADDRESS = 136 "0.0.0.0:" + DEFAULT_RM_PORT; 137 138 /** The actual bind address for the RM.*/ 139 public static final String RM_BIND_HOST = 140 RM_PREFIX + "bind-host"; 141 142 /** The number of threads used to handle applications manager requests.*/ 143 public static final String RM_CLIENT_THREAD_COUNT = 144 RM_PREFIX + "client.thread-count"; 145 public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; 146 147 /** Number of threads used to launch/cleanup AM.*/ 148 public static final String RM_AMLAUNCHER_THREAD_COUNT = 149 RM_PREFIX + "amlauncher.thread-count"; 150 public static final int DEFAULT_RM_AMLAUNCHER_THREAD_COUNT = 50; 151 152 /** Retry times to connect with NM.*/ 153 public static final String RM_NODEMANAGER_CONNECT_RETIRES = 154 RM_PREFIX + "nodemanager-connect-retries"; 155 public static final int DEFAULT_RM_NODEMANAGER_CONNECT_RETIRES = 10; 156 157 /** The Kerberos principal for the resource manager.*/ 158 public static final String RM_PRINCIPAL = 159 RM_PREFIX + "principal"; 160 161 /** The address of the scheduler interface.*/ 162 public static final String RM_SCHEDULER_ADDRESS = 163 RM_PREFIX + "scheduler.address"; 164 public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; 165 public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + 166 DEFAULT_RM_SCHEDULER_PORT; 167 168 /** Miniumum request grant-able by the RM scheduler. */ 169 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 170 YARN_PREFIX + "scheduler.minimum-allocation-mb"; 171 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; 172 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 173 YARN_PREFIX + "scheduler.minimum-allocation-vcores"; 174 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; 175 176 /** Maximum request grant-able by the RM scheduler. */ 177 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 178 YARN_PREFIX + "scheduler.maximum-allocation-mb"; 179 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; 180 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 181 YARN_PREFIX + "scheduler.maximum-allocation-vcores"; 182 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; 183 184 /** Number of threads to handle scheduler interface.*/ 185 public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = 186 RM_PREFIX + "scheduler.client.thread-count"; 187 public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; 188 189 /** If the port should be included or not in the node name. The node name 190 * is used by the scheduler for resource requests allocation location 191 * matching. Typically this is just the hostname, using the port is needed 192 * when using minicluster and specific NM are required.*/ 193 public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME = 194 YARN_PREFIX + "scheduler.include-port-in-node-name"; 195 public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = 196 false; 197 198 /** Whether the RM should enable Reservation System */ 199 public static final String RM_RESERVATION_SYSTEM_ENABLE = RM_PREFIX 200 + "reservation-system.enable"; 201 public static final boolean DEFAULT_RM_RESERVATION_SYSTEM_ENABLE = false; 202 203 /** The class to use as the Reservation System. */ 204 public static final String RM_RESERVATION_SYSTEM_CLASS = RM_PREFIX 205 + "reservation-system.class"; 206 207 /** The PlanFollower for the Reservation System. */ 208 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER = RM_PREFIX 209 + "reservation-system.plan.follower"; 210 211 /** The step size of the Reservation System. */ 212 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP = 213 RM_PREFIX + "reservation-system.planfollower.time-step"; 214 public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP = 215 1000L; 216 217 /** 218 * Enable periodic monitor threads. 219 * @see #RM_SCHEDULER_MONITOR_POLICIES 220 */ 221 public static final String RM_SCHEDULER_ENABLE_MONITORS = 222 RM_PREFIX + "scheduler.monitor.enable"; 223 public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false; 224 225 /** List of SchedulingEditPolicy classes affecting the scheduler. */ 226 public static final String RM_SCHEDULER_MONITOR_POLICIES = 227 RM_PREFIX + "scheduler.monitor.policies"; 228 229 /** The address of the RM web application.*/ 230 public static final String RM_WEBAPP_ADDRESS = 231 RM_PREFIX + "webapp.address"; 232 233 public static final int DEFAULT_RM_WEBAPP_PORT = 8088; 234 public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + 235 DEFAULT_RM_WEBAPP_PORT; 236 237 /** The https address of the RM web application.*/ 238 public static final String RM_WEBAPP_HTTPS_ADDRESS = 239 RM_PREFIX + "webapp.https.address"; 240 public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false; 241 public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml"; 242 243 public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090; 244 public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" 245 + DEFAULT_RM_WEBAPP_HTTPS_PORT; 246 247 public static final String RM_RESOURCE_TRACKER_ADDRESS = 248 RM_PREFIX + "resource-tracker.address"; 249 public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; 250 public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = 251 "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; 252 253 /** The expiry interval for application master reporting.*/ 254 public static final String RM_AM_EXPIRY_INTERVAL_MS = 255 YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; 256 public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; 257 258 /** How long to wait until a node manager is considered dead.*/ 259 public static final String RM_NM_EXPIRY_INTERVAL_MS = 260 YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; 261 public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; 262 263 /** Are acls enabled.*/ 264 public static final String YARN_ACL_ENABLE = 265 YARN_PREFIX + "acl.enable"; 266 public static final boolean DEFAULT_YARN_ACL_ENABLE = false; 267 268 /** ACL of who can be admin of YARN cluster.*/ 269 public static final String YARN_ADMIN_ACL = 270 YARN_PREFIX + "admin.acl"; 271 public static final String DEFAULT_YARN_ADMIN_ACL = "*"; 272 273 /** ACL used in case none is found. Allows nothing. */ 274 public static final String DEFAULT_YARN_APP_ACL = " "; 275 276 /** The address of the RM admin interface.*/ 277 public static final String RM_ADMIN_ADDRESS = 278 RM_PREFIX + "admin.address"; 279 public static final int DEFAULT_RM_ADMIN_PORT = 8033; 280 public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + 281 DEFAULT_RM_ADMIN_PORT; 282 283 /**Number of threads used to handle RM admin interface.*/ 284 public static final String RM_ADMIN_CLIENT_THREAD_COUNT = 285 RM_PREFIX + "admin.client.thread-count"; 286 public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; 287 288 /** 289 * The maximum number of application attempts. 290 * It's a global setting for all application masters. 291 */ 292 public static final String RM_AM_MAX_ATTEMPTS = 293 RM_PREFIX + "am.max-attempts"; 294 public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; 295 296 /** The keytab for the resource manager.*/ 297 public static final String RM_KEYTAB = 298 RM_PREFIX + "keytab"; 299 300 /**The kerberos principal to be used for spnego filter for RM.*/ 301 public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY = 302 RM_PREFIX + "webapp.spnego-principal"; 303 304 /**The kerberos keytab to be used for spnego filter for RM.*/ 305 public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = 306 RM_PREFIX + "webapp.spnego-keytab-file"; 307 308 /** 309 * Flag to enable override of the default kerberos authentication filter with 310 * the RM authentication filter to allow authentication using delegation 311 * tokens(fallback to kerberos if the tokens are missing). Only applicable 312 * when the http authentication type is kerberos. 313 */ 314 public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX 315 + "webapp.delegation-token-auth-filter.enabled"; 316 public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = 317 true; 318 319 /** How long to wait until a container is considered dead.*/ 320 public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 321 RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; 322 public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; 323 324 /** Path to file with nodes to include.*/ 325 public static final String RM_NODES_INCLUDE_FILE_PATH = 326 RM_PREFIX + "nodes.include-path"; 327 public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; 328 329 /** Path to file with nodes to exclude.*/ 330 public static final String RM_NODES_EXCLUDE_FILE_PATH = 331 RM_PREFIX + "nodes.exclude-path"; 332 public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; 333 334 /** Number of threads to handle resource tracker calls.*/ 335 public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 336 RM_PREFIX + "resource-tracker.client.thread-count"; 337 public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; 338 339 /** The class to use as the resource scheduler.*/ 340 public static final String RM_SCHEDULER = 341 RM_PREFIX + "scheduler.class"; 342 343 public static final String DEFAULT_RM_SCHEDULER = 344 "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; 345 346 /** RM set next Heartbeat interval for NM */ 347 public static final String RM_NM_HEARTBEAT_INTERVAL_MS = 348 RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; 349 public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; 350 351 /** Number of worker threads that write the history data. */ 352 public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 353 RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size"; 354 public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 355 10; 356 357 /** 358 * The setting that controls whether yarn system metrics is published on the 359 * timeline server or not by RM. 360 */ 361 public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = 362 RM_PREFIX + "system-metrics-publisher.enabled"; 363 public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false; 364 365 public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 366 RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size"; 367 public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 368 10; 369 370 /** 371 * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the 372 * command being executed to the RM log if this property is true. Commands 373 * may contain sensitive information, such as application or service 374 * passwords, making logging the commands a security risk. In cases where 375 * the cluster may be running applications with such commands, this property 376 * should be set to false. Commands are only logged at the debug level. 377 */ 378 public static final String RM_AMLAUNCHER_LOG_COMMAND = 379 RM_PREFIX + "amlauncher.log.command"; 380 public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false; 381 382 //Delegation token related keys 383 public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = 384 RM_PREFIX + "delegation.key.update-interval"; 385 public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 386 24*60*60*1000; // 1 day 387 public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 388 RM_PREFIX + "delegation.token.renew-interval"; 389 public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 390 24*60*60*1000; // 1 day 391 public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = 392 RM_PREFIX + "delegation.token.max-lifetime"; 393 public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 394 7*24*60*60*1000; // 7 days 395 396 public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; 397 public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; 398 399 public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast"; 400 public static final boolean DEFAULT_YARN_FAIL_FAST = false; 401 402 public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast"; 403 404 @Private 405 public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX 406 + "work-preserving-recovery.enabled"; 407 @Private 408 public static final boolean DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED = 409 false; 410 411 public static final String RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS = 412 RM_PREFIX + "work-preserving-recovery.scheduling-wait-ms"; 413 public static final long DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS = 414 10000; 415 416 /** Zookeeper interaction configs */ 417 public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-"; 418 419 public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; 420 421 public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; 422 public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000; 423 424 public static final String RM_ZK_RETRY_INTERVAL_MS = 425 RM_ZK_PREFIX + "retry-interval-ms"; 426 public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000; 427 428 public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms"; 429 public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000; 430 431 public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl"; 432 public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda"; 433 434 public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth"; 435 436 public static final String ZK_STATE_STORE_PREFIX = 437 RM_PREFIX + "zk-state-store."; 438 439 /** Parent znode path under which ZKRMStateStore will create znodes */ 440 public static final String ZK_RM_STATE_STORE_PARENT_PATH = 441 ZK_STATE_STORE_PREFIX + "parent-path"; 442 public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore"; 443 444 /** Root node ACLs for fencing */ 445 public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL = 446 ZK_STATE_STORE_PREFIX + "root-node.acl"; 447 448 /** HA related configs */ 449 public static final String RM_HA_PREFIX = RM_PREFIX + "ha."; 450 public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled"; 451 public static final boolean DEFAULT_RM_HA_ENABLED = false; 452 453 public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; 454 public static final String RM_HA_ID = RM_HA_PREFIX + "id"; 455 456 /** Store the related configuration files in File System */ 457 public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX 458 + "configuration.file-system-based-store"; 459 public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf"; 460 461 public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX 462 + "configuration.provider-class"; 463 public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = 464 "org.apache.hadoop.yarn.LocalConfigurationProvider"; 465 466 public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX 467 + "authorization-provider"; 468 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = 469 Collections.unmodifiableList(Arrays.asList( 470 RM_ADDRESS, 471 RM_SCHEDULER_ADDRESS, 472 RM_ADMIN_ADDRESS, 473 RM_RESOURCE_TRACKER_ADDRESS, 474 RM_WEBAPP_ADDRESS)); 475 476 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS = 477 Collections.unmodifiableList(Arrays.asList( 478 RM_ADDRESS, 479 RM_SCHEDULER_ADDRESS, 480 RM_ADMIN_ADDRESS, 481 RM_RESOURCE_TRACKER_ADDRESS, 482 RM_WEBAPP_HTTPS_ADDRESS)); 483 484 public static final String AUTO_FAILOVER_PREFIX = 485 RM_HA_PREFIX + "automatic-failover."; 486 487 public static final String AUTO_FAILOVER_ENABLED = 488 AUTO_FAILOVER_PREFIX + "enabled"; 489 public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true; 490 491 public static final String AUTO_FAILOVER_EMBEDDED = 492 AUTO_FAILOVER_PREFIX + "embedded"; 493 public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true; 494 495 public static final String AUTO_FAILOVER_ZK_BASE_PATH = 496 AUTO_FAILOVER_PREFIX + "zk-base-path"; 497 public static final String DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH = 498 "/yarn-leader-election"; 499 500 public static final String CLIENT_FAILOVER_PREFIX = 501 YARN_PREFIX + "client.failover-"; 502 public static final String CLIENT_FAILOVER_PROXY_PROVIDER = 503 CLIENT_FAILOVER_PREFIX + "proxy-provider"; 504 public static final String DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER = 505 "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider"; 506 507 public static final String CLIENT_FAILOVER_MAX_ATTEMPTS = 508 CLIENT_FAILOVER_PREFIX + "max-attempts"; 509 510 public static final String CLIENT_FAILOVER_SLEEPTIME_BASE_MS = 511 CLIENT_FAILOVER_PREFIX + "sleep-base-ms"; 512 513 public static final String CLIENT_FAILOVER_SLEEPTIME_MAX_MS = 514 CLIENT_FAILOVER_PREFIX + "sleep-max-ms"; 515 516 public static final String CLIENT_FAILOVER_RETRIES = 517 CLIENT_FAILOVER_PREFIX + "retries"; 518 public static final int DEFAULT_CLIENT_FAILOVER_RETRIES = 0; 519 520 public static final String CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 521 CLIENT_FAILOVER_PREFIX + "retries-on-socket-timeouts"; 522 public static final int 523 DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0; 524 525 //////////////////////////////// 526 // RM state store configs 527 //////////////////////////////// 528 /** The class to use as the persistent store.*/ 529 public static final String RM_STORE = RM_PREFIX + "store.class"; 530 531 /** URI for FileSystemRMStateStore */ 532 public static final String FS_RM_STATE_STORE_URI = RM_PREFIX 533 + "fs.state-store.uri"; 534 public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX 535 + "fs.state-store.retry-policy-spec"; 536 public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC = 537 "2000, 500"; 538 539 /** The maximum number of completed applications RM keeps. */ 540 public static final String RM_MAX_COMPLETED_APPLICATIONS = 541 RM_PREFIX + "max-completed-applications"; 542 public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; 543 544 /** 545 * The maximum number of completed applications RM state store keeps, by 546 * default equals to DEFAULT_RM_MAX_COMPLETED_APPLICATIONS 547 */ 548 public static final String RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = 549 RM_PREFIX + "state-store.max-completed-applications"; 550 public static final int DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = 551 DEFAULT_RM_MAX_COMPLETED_APPLICATIONS; 552 553 /** Default application name */ 554 public static final String DEFAULT_APPLICATION_NAME = "N/A"; 555 556 /** Default application type */ 557 public static final String DEFAULT_APPLICATION_TYPE = "YARN"; 558 559 /** Default application type length */ 560 public static final int APPLICATION_TYPE_LENGTH = 20; 561 562 /** Default queue name */ 563 public static final String DEFAULT_QUEUE_NAME = "default"; 564 565 /** 566 * Buckets (in minutes) for the number of apps running in each queue. 567 */ 568 public static final String RM_METRICS_RUNTIME_BUCKETS = 569 RM_PREFIX + "metrics.runtime.buckets"; 570 571 /** 572 * Default sizes of the runtime metric buckets in minutes. 573 */ 574 public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = 575 "60,300,1440"; 576 577 public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX 578 + "am-rm-tokens.master-key-rolling-interval-secs"; 579 580 public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 581 24 * 60 * 60; 582 583 public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 584 RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; 585 586 public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 587 24 * 60 * 60; 588 589 public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 590 RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs"; 591 592 public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 593 24 * 60 * 60; 594 595 public static final String RM_NODEMANAGER_MINIMUM_VERSION = 596 RM_PREFIX + "nodemanager.minimum.version"; 597 598 public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION = 599 "NONE"; 600 601 /** 602 * RM proxy users' prefix 603 */ 604 public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser."; 605 606 //////////////////////////////// 607 // Node Manager Configs 608 //////////////////////////////// 609 610 /** Prefix for all node manager configs.*/ 611 public static final String NM_PREFIX = "yarn.nodemanager."; 612 613 /** Environment variables that will be sent to containers.*/ 614 public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; 615 public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; 616 617 /** Environment variables that containers may override rather than use NodeManager's default.*/ 618 public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; 619 public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",", 620 Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(), 621 ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), 622 ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), 623 ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), 624 ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); 625 626 /** address of node manager IPC.*/ 627 public static final String NM_ADDRESS = NM_PREFIX + "address"; 628 public static final int DEFAULT_NM_PORT = 0; 629 public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" 630 + DEFAULT_NM_PORT; 631 632 /** The actual bind address or the NM.*/ 633 public static final String NM_BIND_HOST = 634 NM_PREFIX + "bind-host"; 635 636 /** who will execute(launch) the containers.*/ 637 public static final String NM_CONTAINER_EXECUTOR = 638 NM_PREFIX + "container-executor.class"; 639 640 /** 641 * Adjustment to make to the container os scheduling priority. 642 * The valid values for this could vary depending on the platform. 643 * On Linux, higher values mean run the containers at a less 644 * favorable priority than the NM. 645 * The value specified is an int. 646 */ 647 public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 648 NM_PREFIX + "container-executor.os.sched.priority.adjustment"; 649 public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; 650 651 /** Number of threads container manager uses.*/ 652 public static final String NM_CONTAINER_MGR_THREAD_COUNT = 653 NM_PREFIX + "container-manager.thread-count"; 654 public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; 655 656 /** Number of threads used in cleanup.*/ 657 public static final String NM_DELETE_THREAD_COUNT = 658 NM_PREFIX + "delete.thread-count"; 659 public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; 660 661 /** Keytab for NM.*/ 662 public static final String NM_KEYTAB = NM_PREFIX + "keytab"; 663 664 /**List of directories to store localized files in.*/ 665 public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; 666 public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; 667 668 /** 669 * Number of files in each localized directories 670 * Avoid tuning this too low. 671 */ 672 public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 673 NM_PREFIX + "local-cache.max-files-per-directory"; 674 public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; 675 676 /** Address where the localizer IPC is.*/ 677 public static final String NM_LOCALIZER_ADDRESS = 678 NM_PREFIX + "localizer.address"; 679 public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; 680 public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + 681 DEFAULT_NM_LOCALIZER_PORT; 682 683 /** Interval in between cache cleanups.*/ 684 public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 685 NM_PREFIX + "localizer.cache.cleanup.interval-ms"; 686 public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 687 10 * 60 * 1000; 688 689 /** 690 * Target size of localizer cache in MB, per nodemanager. It is a target 691 * retention size that only includes resources with PUBLIC and PRIVATE 692 * visibility and excludes resources with APPLICATION visibility 693 */ 694 public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 695 NM_PREFIX + "localizer.cache.target-size-mb"; 696 public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; 697 698 /** Number of threads to handle localization requests.*/ 699 public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = 700 NM_PREFIX + "localizer.client.thread-count"; 701 public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; 702 703 /** Number of threads to use for localization fetching.*/ 704 public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = 705 NM_PREFIX + "localizer.fetch.thread-count"; 706 public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; 707 708 /** Where to store container logs.*/ 709 public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; 710 public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; 711 712 /** The number of threads to handle log aggregation in node manager. */ 713 public static final String NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 714 NM_PREFIX + "logaggregation.threadpool-size-max"; 715 public static final int DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 100; 716 717 /** Default permissions for container logs. */ 718 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX = 719 NM_PREFIX + "default-container-executor."; 720 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS = 721 NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX + "log-dirs.permissions"; 722 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT = "710"; 723 724 public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = 725 NM_PREFIX + "resourcemanager.minimum.version"; 726 public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; 727 728 /** Interval at which the delayed token removal thread runs */ 729 public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 730 RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; 731 public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 732 30000l; 733 734 /** Delegation Token renewer thread count */ 735 public static final String RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 736 RM_PREFIX + "delegation-token-renewer.thread-count"; 737 public static final int DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 50; 738 739 public static final String RM_PROXY_USER_PRIVILEGES_ENABLED = RM_PREFIX 740 + "proxy-user-privileges.enabled"; 741 public static boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false; 742 743 /** 744 * How many diagnostics/failure messages can be saved in RM for 745 * log aggregation. It also defines the number of diagnostics/failure 746 * messages can be shown in log aggregation web ui. 747 */ 748 public static final String RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY = 749 RM_PREFIX + "max-log-aggregation-diagnostics-in-memory"; 750 public static final int DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY = 751 10; 752 753 /** Whether to enable log aggregation */ 754 public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX 755 + "log-aggregation-enable"; 756 public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; 757 758 /** 759 * How long to wait before deleting aggregated logs, -1 disables. 760 * Be careful set this too small and you will spam the name node. 761 */ 762 public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX 763 + "log-aggregation.retain-seconds"; 764 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; 765 766 /** 767 * How long to wait between aggregated log retention checks. If set to 768 * a value <= 0 then the value is computed as one-tenth of the log retention 769 * setting. Be careful set this too small and you will spam the name node. 770 */ 771 public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = 772 YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; 773 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; 774 775 /** 776 * How long for ResourceManager to wait for NodeManager to report its 777 * log aggregation status. If waiting time of which the log aggregation status 778 * is reported from NodeManager exceeds the configured value, RM will report 779 * log aggregation status for this NodeManager as TIME_OUT 780 */ 781 public static final String LOG_AGGREGATION_STATUS_TIME_OUT_MS = 782 YARN_PREFIX + "log-aggregation-status.time-out.ms"; 783 public static final long DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS 784 = 10 * 60 * 1000; 785 786 /** 787 * Number of seconds to retain logs on the NodeManager. Only applicable if Log 788 * aggregation is disabled 789 */ 790 public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX 791 + "log.retain-seconds"; 792 public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60; 793 794 /** 795 * Define how often NMs wake up and upload log files 796 */ 797 public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = 798 NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; 799 public static final long 800 DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; 801 /** 802 * Number of threads used in log cleanup. Only applicable if Log aggregation 803 * is disabled 804 */ 805 public static final String NM_LOG_DELETION_THREADS_COUNT = 806 NM_PREFIX + "log.deletion-threads-count"; 807 public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; 808 809 /** Where to aggregate logs to.*/ 810 public static final String NM_REMOTE_APP_LOG_DIR = 811 NM_PREFIX + "remote-app-log-dir"; 812 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; 813 814 /** 815 * The remote log dir will be created at 816 * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} 817 */ 818 public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = 819 NM_PREFIX + "remote-app-log-dir-suffix"; 820 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; 821 822 public static final String YARN_LOG_SERVER_URL = 823 YARN_PREFIX + "log.server.url"; 824 825 public static final String YARN_TRACKING_URL_GENERATOR = 826 YARN_PREFIX + "tracking.url.generator"; 827 828 /** Amount of memory in GB that can be allocated for containers.*/ 829 public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; 830 public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; 831 832 /** Specifies whether physical memory check is enabled. */ 833 public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX 834 + "pmem-check-enabled"; 835 public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; 836 837 /** Specifies whether physical memory check is enabled. */ 838 public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX 839 + "vmem-check-enabled"; 840 public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = false; 841 842 /** Conversion ratio for physical memory to virtual memory. */ 843 public static final String NM_VMEM_PMEM_RATIO = 844 NM_PREFIX + "vmem-pmem-ratio"; 845 public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; 846 847 /** Number of Virtual CPU Cores which can be allocated for containers.*/ 848 public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores"; 849 public static final int DEFAULT_NM_VCORES = 8; 850 851 /** Percentage of overall CPU which can be allocated for containers. */ 852 public static final String NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 853 NM_PREFIX + "resource.percentage-physical-cpu-limit"; 854 public static final int DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 855 100; 856 857 /** 858 * Prefix for disk configurations. Work in progress: This configuration 859 * parameter may be changed/removed in the future. 860 */ 861 @Private 862 public static final String NM_DISK_RESOURCE_PREFIX = NM_PREFIX 863 + "resource.disk."; 864 /** 865 * This setting controls if resource handling for disk operations is enabled. 866 * Work in progress: This configuration parameter may be changed/removed in 867 * the future 868 */ 869 @Private 870 public static final String NM_DISK_RESOURCE_ENABLED = NM_DISK_RESOURCE_PREFIX 871 + "enabled"; 872 /** Disk as a resource is disabled by default. **/ 873 @Private 874 public static final boolean DEFAULT_NM_DISK_RESOURCE_ENABLED = false; 875 876 public static final String NM_NETWORK_RESOURCE_PREFIX = NM_PREFIX 877 + "resource.network."; 878 879 /** 880 * This setting controls if resource handling for network bandwidth is 881 * enabled. Work in progress: This configuration parameter may be 882 * changed/removed in the future 883 */ 884 @Private 885 public static final String NM_NETWORK_RESOURCE_ENABLED = 886 NM_NETWORK_RESOURCE_PREFIX + "enabled"; 887 /** Network as a resource is disabled by default. **/ 888 @Private 889 public static final boolean DEFAULT_NM_NETWORK_RESOURCE_ENABLED = false; 890 891 /** 892 * Specifies the interface to be used for applying network throttling rules. 893 * Work in progress: This configuration parameter may be changed/removed in 894 * the future 895 */ 896 @Private 897 public static final String NM_NETWORK_RESOURCE_INTERFACE = 898 NM_NETWORK_RESOURCE_PREFIX + "interface"; 899 @Private 900 public static final String DEFAULT_NM_NETWORK_RESOURCE_INTERFACE = "eth0"; 901 902 /** 903 * Specifies the total available outbound bandwidth on the node. Work in 904 * progress: This configuration parameter may be changed/removed in the future 905 */ 906 @Private 907 public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT = 908 NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-mbit"; 909 @Private 910 public static final int DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT = 911 1000; 912 913 /** 914 * Specifies the total outbound bandwidth available to YARN containers. 915 * defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified. 916 * Work in progress: This configuration parameter may be changed/removed in 917 * the future 918 */ 919 @Private 920 public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT = 921 NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-yarn-mbit"; 922 923 /** NM Webapp address.**/ 924 public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; 925 public static final int DEFAULT_NM_WEBAPP_PORT = 8042; 926 public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + 927 DEFAULT_NM_WEBAPP_PORT; 928 929 /** NM Webapp https address.**/ 930 public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX 931 + "webapp.https.address"; 932 public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044; 933 public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" 934 + DEFAULT_NM_WEBAPP_HTTPS_PORT; 935 936 /** How often to monitor containers.*/ 937 public final static String NM_CONTAINER_MON_INTERVAL_MS = 938 NM_PREFIX + "container-monitor.interval-ms"; 939 public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; 940 941 /** Class that calculates containers current resource utilization.*/ 942 public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = 943 NM_PREFIX + "container-monitor.resource-calculator.class"; 944 /** Class that calculates process tree resource utilization.*/ 945 public static final String NM_CONTAINER_MON_PROCESS_TREE = 946 NM_PREFIX + "container-monitor.process-tree.class"; 947 public static final String PROCFS_USE_SMAPS_BASED_RSS_ENABLED = NM_PREFIX + 948 "container-monitor.procfs-tree.smaps-based-rss.enabled"; 949 public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED = 950 false; 951 952 /** Enable/disable container metrics. */ 953 @Private 954 public static final String NM_CONTAINER_METRICS_ENABLE = 955 NM_PREFIX + "container-metrics.enable"; 956 @Private 957 public static final boolean DEFAULT_NM_CONTAINER_METRICS_ENABLE = true; 958 959 /** Container metrics flush period. -1 for flush on completion. */ 960 @Private 961 public static final String NM_CONTAINER_METRICS_PERIOD_MS = 962 NM_PREFIX + "container-metrics.period-ms"; 963 @Private 964 public static final int DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS = -1; 965 966 /** The delay time ms to unregister container metrics after completion. */ 967 @Private 968 public static final String NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS = 969 NM_PREFIX + "container-metrics.unregister-delay-ms"; 970 @Private 971 public static final int DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS = 972 10000; 973 974 /** Prefix for all node manager disk health checker configs. */ 975 private static final String NM_DISK_HEALTH_CHECK_PREFIX = 976 "yarn.nodemanager.disk-health-checker."; 977 /** 978 * Enable/Disable disks' health checker. Default is true. An expert level 979 * configuration property. 980 */ 981 public static final String NM_DISK_HEALTH_CHECK_ENABLE = 982 NM_DISK_HEALTH_CHECK_PREFIX + "enable"; 983 /** Frequency of running disks' health checker. */ 984 public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = 985 NM_DISK_HEALTH_CHECK_PREFIX + "interval-ms"; 986 /** By default, disks' health is checked every 2 minutes. */ 987 public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = 988 2 * 60 * 1000; 989 990 /** 991 * The minimum fraction of number of disks to be healthy for the nodemanager 992 * to launch new containers. This applies to nm-local-dirs and nm-log-dirs. 993 */ 994 public static final String NM_MIN_HEALTHY_DISKS_FRACTION = 995 NM_DISK_HEALTH_CHECK_PREFIX + "min-healthy-disks"; 996 /** 997 * By default, at least 25% of disks are to be healthy to say that the node is 998 * healthy in terms of disks. 999 */ 1000 public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION = 0.25F; 1001 1002 /** 1003 * The maximum percentage of disk space that can be used after which a disk is 1004 * marked as offline. Values can range from 0.0 to 100.0. If the value is 1005 * greater than or equal to 100, NM will check for full disk. This applies to 1006 * nm-local-dirs and nm-log-dirs. 1007 */ 1008 public static final String NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 1009 NM_DISK_HEALTH_CHECK_PREFIX + "max-disk-utilization-per-disk-percentage"; 1010 /** 1011 * By default, 90% of the disk can be used before it is marked as offline. 1012 */ 1013 public static final float DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 1014 90.0F; 1015 1016 /** 1017 * The low threshold percentage of disk space used when an offline disk is 1018 * marked as online. Values can range from 0.0 to 100.0. The value shouldn't 1019 * be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is 1020 * more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be 1021 * set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. 1022 * This applies to nm-local-dirs and nm-log-dirs. 1023 */ 1024 public static final String NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE = 1025 NM_DISK_HEALTH_CHECK_PREFIX + 1026 "disk-utilization-watermark-low-per-disk-percentage"; 1027 1028 /** 1029 * The minimum space that must be available on a local dir for it to be used. 1030 * This applies to nm-local-dirs and nm-log-dirs. 1031 */ 1032 public static final String NM_MIN_PER_DISK_FREE_SPACE_MB = 1033 NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb"; 1034 /** 1035 * By default, all of the disk can be used before it is marked as offline. 1036 */ 1037 public static final long DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB = 0; 1038 1039 /** Frequency of running node health script.*/ 1040 public static final String NM_HEALTH_CHECK_INTERVAL_MS = 1041 NM_PREFIX + "health-checker.interval-ms"; 1042 public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; 1043 1044 /** Health check script time out period.*/ 1045 public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 1046 NM_PREFIX + "health-checker.script.timeout-ms"; 1047 public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 1048 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; 1049 1050 /** The health check script to run.*/ 1051 public static final String NM_HEALTH_CHECK_SCRIPT_PATH = 1052 NM_PREFIX + "health-checker.script.path"; 1053 1054 /** The arguments to pass to the health check script.*/ 1055 public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 1056 NM_PREFIX + "health-checker.script.opts"; 1057 1058 /** The JVM options used on forking ContainerLocalizer process 1059 by container executor. */ 1060 public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY = 1061 NM_PREFIX + "container-localizer.java.opts"; 1062 public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = 1063 "-Xmx256m"; 1064 1065 /** The Docker image name(For DockerContainerExecutor).*/ 1066 public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME = 1067 NM_PREFIX + "docker-container-executor.image-name"; 1068 1069 /** The name of the docker executor (For DockerContainerExecutor).*/ 1070 public static final String NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = 1071 NM_PREFIX + "docker-container-executor.exec-name"; 1072 1073 /** The default docker executor (For DockerContainerExecutor).*/ 1074 public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = 1075 "/usr/bin/docker"; 1076 1077 /** The path to the Linux container executor.*/ 1078 public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = 1079 NM_PREFIX + "linux-container-executor.path"; 1080 1081 /** 1082 * The UNIX group that the linux-container-executor should run as. 1083 * This is intended to be set as part of container-executor.cfg. 1084 */ 1085 public static final String NM_LINUX_CONTAINER_GROUP = 1086 NM_PREFIX + "linux-container-executor.group"; 1087 1088 /** 1089 * If linux-container-executor should limit itself to one user 1090 * when running in non-secure mode. 1091 */ 1092 public static final String NM_NONSECURE_MODE_LIMIT_USERS= NM_PREFIX + 1093 "linux-container-executor.nonsecure-mode.limit-users"; 1094 1095 public static final boolean DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS = true; 1096 1097 /** 1098 * The UNIX user that containers will run as when Linux-container-executor 1099 * is used in nonsecure mode (a use case for this is using cgroups). 1100 */ 1101 public static final String NM_NONSECURE_MODE_LOCAL_USER_KEY = NM_PREFIX + 1102 "linux-container-executor.nonsecure-mode.local-user"; 1103 1104 public static final String DEFAULT_NM_NONSECURE_MODE_LOCAL_USER = "nobody"; 1105 1106 /** 1107 * The allowed pattern for UNIX user names enforced by 1108 * Linux-container-executor when used in nonsecure mode (use case for this 1109 * is using cgroups). The default value is taken from /usr/sbin/adduser 1110 */ 1111 public static final String NM_NONSECURE_MODE_USER_PATTERN_KEY = NM_PREFIX + 1112 "linux-container-executor.nonsecure-mode.user-pattern"; 1113 1114 public static final String DEFAULT_NM_NONSECURE_MODE_USER_PATTERN = 1115 "^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$"; 1116 1117 /** The type of resource enforcement to use with the 1118 * linux container executor. 1119 */ 1120 public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = 1121 NM_PREFIX + "linux-container-executor.resources-handler.class"; 1122 1123 /** The path the linux container executor should use for cgroups */ 1124 public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = 1125 NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; 1126 1127 /** Whether the linux container executor should mount cgroups if not found */ 1128 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = 1129 NM_PREFIX + "linux-container-executor.cgroups.mount"; 1130 1131 /** Where the linux container executor should mount cgroups if not found */ 1132 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = 1133 NM_PREFIX + "linux-container-executor.cgroups.mount-path"; 1134 1135 /** 1136 * Whether the apps should run in strict resource usage mode(not allowed to 1137 * use spare CPU) 1138 */ 1139 public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = 1140 NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; 1141 public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = 1142 false; 1143 1144 1145 1146 /** 1147 * Interval of time the linux container executor should try cleaning up 1148 * cgroups entry when cleaning up a container. This is required due to what 1149 * it seems a race condition because the SIGTERM/SIGKILL is asynch. 1150 */ 1151 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT = 1152 NM_PREFIX + "linux-container-executor.cgroups.delete-timeout-ms"; 1153 1154 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT = 1155 1000; 1156 1157 /** 1158 * Delay between attempts to remove linux cgroup. 1159 */ 1160 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY = 1161 NM_PREFIX + "linux-container-executor.cgroups.delete-delay-ms"; 1162 1163 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY = 1164 20; 1165 1166 /** 1167 /* The Windows group that the windows-secure-container-executor should run as. 1168 */ 1169 public static final String NM_WINDOWS_SECURE_CONTAINER_GROUP = 1170 NM_PREFIX + "windows-secure-container-executor.group"; 1171 1172 /** T-file compression types used to compress aggregated logs.*/ 1173 public static final String NM_LOG_AGG_COMPRESSION_TYPE = 1174 NM_PREFIX + "log-aggregation.compression-type"; 1175 public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; 1176 1177 /** The kerberos principal for the node manager.*/ 1178 public static final String NM_PRINCIPAL = 1179 NM_PREFIX + "principal"; 1180 1181 public static final String NM_AUX_SERVICES = 1182 NM_PREFIX + "aux-services"; 1183 1184 public static final String NM_AUX_SERVICE_FMT = 1185 NM_PREFIX + "aux-services.%s.class"; 1186 1187 public static final String NM_USER_HOME_DIR = 1188 NM_PREFIX + "user-home-dir"; 1189 1190 /**The kerberos principal to be used for spnego filter for NM.*/ 1191 public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY = 1192 NM_PREFIX + "webapp.spnego-principal"; 1193 1194 /**The kerberos keytab to be used for spnego filter for NM.*/ 1195 public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = 1196 NM_PREFIX + "webapp.spnego-keytab-file"; 1197 1198 public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; 1199 1200 public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery."; 1201 public static final String NM_RECOVERY_ENABLED = 1202 NM_RECOVERY_PREFIX + "enabled"; 1203 public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false; 1204 1205 public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir"; 1206 1207 /** The time in seconds between full compactions of the NM state database. 1208 * Setting the interval to zero disables the full compaction cycles. 1209 */ 1210 public static final String NM_RECOVERY_COMPACTION_INTERVAL_SECS = 1211 NM_RECOVERY_PREFIX + "compaction-interval-secs"; 1212 public static final int DEFAULT_NM_RECOVERY_COMPACTION_INTERVAL_SECS = 3600; 1213 1214 //////////////////////////////// 1215 // Web Proxy Configs 1216 //////////////////////////////// 1217 public static final String PROXY_PREFIX = "yarn.web-proxy."; 1218 1219 /** The kerberos principal for the proxy.*/ 1220 public static final String PROXY_PRINCIPAL = 1221 PROXY_PREFIX + "principal"; 1222 1223 /** Keytab for Proxy.*/ 1224 public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; 1225 1226 /** The address for the web proxy.*/ 1227 public static final String PROXY_ADDRESS = 1228 PROXY_PREFIX + "address"; 1229 public static final int DEFAULT_PROXY_PORT = 9099; 1230 public static final String DEFAULT_PROXY_ADDRESS = 1231 "0.0.0.0:" + DEFAULT_PROXY_PORT; 1232 1233 /** 1234 * YARN Service Level Authorization 1235 */ 1236 public static final String 1237 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL = 1238 "security.resourcetracker.protocol.acl"; 1239 public static final String 1240 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL = 1241 "security.applicationclient.protocol.acl"; 1242 public static final String 1243 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL = 1244 "security.resourcemanager-administration.protocol.acl"; 1245 public static final String 1246 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL = 1247 "security.applicationmaster.protocol.acl"; 1248 1249 public static final String 1250 YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL = 1251 "security.containermanagement.protocol.acl"; 1252 public static final String 1253 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = 1254 "security.resourcelocalizer.protocol.acl"; 1255 1256 public static final String 1257 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL = 1258 "security.applicationhistory.protocol.acl"; 1259 1260 /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL 1261 * to a running container */ 1262 public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 1263 NM_PREFIX + "sleep-delay-before-sigkill.ms"; 1264 public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 1265 250; 1266 1267 /** Max time to wait for a process to come up when trying to cleanup 1268 * container resources */ 1269 public static final String NM_PROCESS_KILL_WAIT_MS = 1270 NM_PREFIX + "process-kill-wait.ms"; 1271 public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 1272 2000; 1273 1274 /** Max time to wait to establish a connection to RM */ 1275 public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1276 RM_PREFIX + "connect.max-wait.ms"; 1277 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1278 15 * 60 * 1000; 1279 1280 /** Time interval between each attempt to connect to RM */ 1281 public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = 1282 RM_PREFIX + "connect.retry-interval.ms"; 1283 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS 1284 = 30 * 1000; 1285 1286 /** 1287 * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH 1288 * entries 1289 */ 1290 public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX 1291 + "application.classpath"; 1292 1293 /** 1294 * Default platform-agnostic CLASSPATH for YARN applications. A 1295 * comma-separated list of CLASSPATH entries. The parameter expansion marker 1296 * will be replaced with real parameter expansion marker ('%' for Windows and 1297 * '$' for Linux) by NodeManager on container launch. For example: {{VAR}} 1298 * will be replaced as $VAR on Linux, and %VAR% on Windows. 1299 */ 1300 @Public 1301 @Unstable 1302 public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= { 1303 ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(), 1304 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() 1305 + "/share/hadoop/common/*", 1306 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() 1307 + "/share/hadoop/common/lib/*", 1308 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() 1309 + "/share/hadoop/hdfs/*", 1310 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() 1311 + "/share/hadoop/hdfs/lib/*", 1312 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() 1313 + "/share/hadoop/yarn/*", 1314 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() 1315 + "/share/hadoop/yarn/lib/*" }; 1316 /** 1317 * <p> 1318 * Default platform-specific CLASSPATH for YARN applications. A 1319 * comma-separated list of CLASSPATH entries constructed based on the client 1320 * OS environment expansion syntax. 1321 * </p> 1322 * <p> 1323 * Note: Use {@link DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for 1324 * cross-platform practice i.e. submit an application from a Windows client to 1325 * a Linux/Unix server or vice versa. 1326 * </p> 1327 */ 1328 public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { 1329 ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), 1330 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 1331 + "/share/hadoop/common/*", 1332 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 1333 + "/share/hadoop/common/lib/*", 1334 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 1335 + "/share/hadoop/hdfs/*", 1336 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 1337 + "/share/hadoop/hdfs/lib/*", 1338 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 1339 + "/share/hadoop/yarn/*", 1340 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 1341 + "/share/hadoop/yarn/lib/*" }; 1342 1343 /** Container temp directory */ 1344 public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; 1345 1346 public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX 1347 + "is.minicluster"; 1348 1349 public static final String YARN_MC_PREFIX = YARN_PREFIX + "minicluster."; 1350 1351 /** Whether to use fixed ports with the minicluster. */ 1352 public static final String YARN_MINICLUSTER_FIXED_PORTS = 1353 YARN_MC_PREFIX + "fixed.ports"; 1354 1355 /** 1356 * Default is false to be able to run tests concurrently without port 1357 * conflicts. 1358 */ 1359 public static final boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; 1360 1361 /** 1362 * Whether the NM should use RPC to connect to the RM. Default is false. 1363 * Can be set to true only when using fixed ports. 1364 */ 1365 public static final String YARN_MINICLUSTER_USE_RPC = YARN_MC_PREFIX + "use-rpc"; 1366 public static final boolean DEFAULT_YARN_MINICLUSTER_USE_RPC = false; 1367 1368 /** 1369 * Whether users are explicitly trying to control resource monitoring 1370 * configuration for the MiniYARNCluster. Disabled by default. 1371 */ 1372 public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = 1373 YARN_MC_PREFIX + "control-resource-monitoring"; 1374 public static final boolean 1375 DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false; 1376 1377 /** The log directory for the containers */ 1378 public static final String YARN_APP_CONTAINER_LOG_DIR = 1379 YARN_PREFIX + "app.container.log.dir"; 1380 1381 public static final String YARN_APP_CONTAINER_LOG_SIZE = 1382 YARN_PREFIX + "app.container.log.filesize"; 1383 1384 public static final String YARN_APP_CONTAINER_LOG_BACKUPS = 1385 YARN_PREFIX + "app.container.log.backups"; 1386 1387 /** Directory for fail flag files */ 1388 @Private 1389 public static final String YARN_AM_FAILURE_FLAG_DIR = 1390 YARN_PREFIX + "am-failure.flag.dir"; 1391 @Private 1392 public static final String DEFAULT_YARN_AM_FAILURE_FLAG_DIR = 1393 "/tmp/hadoop-yarn/fail"; 1394 1395 //////////////////////////////// 1396 // Timeline Service Configs 1397 //////////////////////////////// 1398 1399 public static final String TIMELINE_SERVICE_PREFIX = 1400 YARN_PREFIX + "timeline-service."; 1401 1402 1403 // mark app-history related configs @Private as application history is going 1404 // to be integrated into the timeline service 1405 @Private 1406 public static final String APPLICATION_HISTORY_PREFIX = 1407 TIMELINE_SERVICE_PREFIX + "generic-application-history."; 1408 1409 /** 1410 * The setting that controls whether application history service is 1411 * enabled or not. 1412 */ 1413 @Private 1414 public static final String APPLICATION_HISTORY_ENABLED = 1415 APPLICATION_HISTORY_PREFIX + "enabled"; 1416 @Private 1417 public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false; 1418 1419 /** Application history store class */ 1420 @Private 1421 public static final String APPLICATION_HISTORY_STORE = 1422 APPLICATION_HISTORY_PREFIX + "store-class"; 1423 1424 /** URI for FileSystemApplicationHistoryStore */ 1425 @Private 1426 public static final String FS_APPLICATION_HISTORY_STORE_URI = 1427 APPLICATION_HISTORY_PREFIX + "fs-history-store.uri"; 1428 1429 /** T-file compression types used to compress history data.*/ 1430 @Private 1431 public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = 1432 APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type"; 1433 @Private 1434 public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = 1435 "none"; 1436 1437 /** The setting that controls whether timeline service is enabled or not. */ 1438 public static final String TIMELINE_SERVICE_ENABLED = 1439 TIMELINE_SERVICE_PREFIX + "enabled"; 1440 public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = false; 1441 1442 /** host:port address for timeline service RPC APIs. */ 1443 public static final String TIMELINE_SERVICE_ADDRESS = 1444 TIMELINE_SERVICE_PREFIX + "address"; 1445 public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200; 1446 public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:" 1447 + DEFAULT_TIMELINE_SERVICE_PORT; 1448 1449 /** The listening endpoint for the timeline service application.*/ 1450 public static final String TIMELINE_SERVICE_BIND_HOST = 1451 TIMELINE_SERVICE_PREFIX + "bind-host"; 1452 1453 /** The number of threads to handle client RPC API requests. */ 1454 public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT = 1455 TIMELINE_SERVICE_PREFIX + "handler-thread-count"; 1456 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10; 1457 1458 1459 /** The address of the timeline service web application.*/ 1460 public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS = 1461 TIMELINE_SERVICE_PREFIX + "webapp.address"; 1462 1463 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188; 1464 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS = 1465 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT; 1466 1467 /** The https address of the timeline service web application.*/ 1468 public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = 1469 TIMELINE_SERVICE_PREFIX + "webapp.https.address"; 1470 1471 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190; 1472 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = 1473 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; 1474 1475 /** Timeline service store class */ 1476 public static final String TIMELINE_SERVICE_STORE = 1477 TIMELINE_SERVICE_PREFIX + "store-class"; 1478 1479 /** Timeline service enable data age off */ 1480 public static final String TIMELINE_SERVICE_TTL_ENABLE = 1481 TIMELINE_SERVICE_PREFIX + "ttl-enable"; 1482 1483 /** Timeline service length of time to retain data */ 1484 public static final String TIMELINE_SERVICE_TTL_MS = 1485 TIMELINE_SERVICE_PREFIX + "ttl-ms"; 1486 1487 public static final long DEFAULT_TIMELINE_SERVICE_TTL_MS = 1488 1000 * 60 * 60 * 24 * 7; 1489 1490 public static final String TIMELINE_SERVICE_LEVELDB_PREFIX = 1491 TIMELINE_SERVICE_PREFIX + "leveldb-timeline-store."; 1492 1493 /** Timeline service leveldb path */ 1494 public static final String TIMELINE_SERVICE_LEVELDB_PATH = 1495 TIMELINE_SERVICE_LEVELDB_PREFIX + "path"; 1496 1497 /** Timeline service leveldb read cache (uncompressed blocks) */ 1498 public static final String TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE = 1499 TIMELINE_SERVICE_LEVELDB_PREFIX + "read-cache-size"; 1500 1501 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE = 1502 100 * 1024 * 1024; 1503 1504 /** Timeline service leveldb start time read cache (number of entities) */ 1505 public static final String 1506 TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 1507 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-read-cache-size"; 1508 1509 public static final int 1510 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 10000; 1511 1512 /** Timeline service leveldb start time write cache (number of entities) */ 1513 public static final String 1514 TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 1515 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-write-cache-size"; 1516 1517 public static final int 1518 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 10000; 1519 1520 /** Timeline service leveldb interval to wait between deletion rounds */ 1521 public static final String TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1522 TIMELINE_SERVICE_LEVELDB_PREFIX + "ttl-interval-ms"; 1523 1524 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1525 1000 * 60 * 5; 1526 1527 /** The Kerberos principal for the timeline server.*/ 1528 public static final String TIMELINE_SERVICE_PRINCIPAL = 1529 TIMELINE_SERVICE_PREFIX + "principal"; 1530 1531 /** The Kerberos keytab for the timeline server.*/ 1532 public static final String TIMELINE_SERVICE_KEYTAB = 1533 TIMELINE_SERVICE_PREFIX + "keytab"; 1534 1535 /** Enables cross origin support for timeline server.*/ 1536 public static final String TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED = 1537 TIMELINE_SERVICE_PREFIX + "http-cross-origin.enabled"; 1538 1539 /** Default value for cross origin support for timeline server.*/ 1540 public static final boolean 1541 TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false; 1542 1543 /** Timeline client settings */ 1544 public static final String TIMELINE_SERVICE_CLIENT_PREFIX = 1545 TIMELINE_SERVICE_PREFIX + "client."; 1546 1547 /** Timeline client call, max retries (-1 means no limit) */ 1548 public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 1549 TIMELINE_SERVICE_CLIENT_PREFIX + "max-retries"; 1550 1551 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30; 1552 1553 /** Timeline client call, retry interval */ 1554 public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1555 TIMELINE_SERVICE_CLIENT_PREFIX + "retry-interval-ms"; 1556 1557 public static final long 1558 DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000; 1559 1560 //////////////////////////////// 1561 // Other Configs 1562 //////////////////////////////// 1563 1564 /** 1565 * Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead. 1566 * The interval of the yarn client's querying application state after 1567 * application submission. The unit is millisecond. 1568 */ 1569 @Deprecated 1570 public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = 1571 YARN_PREFIX + "client.app-submission.poll-interval"; 1572 1573 /** 1574 * The interval that the yarn client library uses to poll the completion 1575 * status of the asynchronous API of application client protocol. 1576 */ 1577 public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS = 1578 YARN_PREFIX + "client.application-client-protocol.poll-interval-ms"; 1579 public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS = 1580 200; 1581 1582 /** 1583 * The duration that the yarn client library waits, cumulatively across polls, 1584 * for an expected state change to occur. Defaults to -1, which indicates no 1585 * limit. 1586 */ 1587 public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS = 1588 YARN_PREFIX + "client.application-client-protocol.poll-timeout-ms"; 1589 public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS = 1590 -1; 1591 1592 /** 1593 * Max number of threads in NMClientAsync to process container management 1594 * events 1595 */ 1596 public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 1597 YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size"; 1598 public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500; 1599 1600 /** 1601 * Maximum number of proxy connections to cache for node managers. If set 1602 * to a value greater than zero then the cache is enabled and the NMClient 1603 * and MRAppMaster will cache the specified number of node manager proxies. 1604 * There will be at max one proxy per node manager. Ex. configuring it to a 1605 * value of 5 will make sure that client will at max have 5 proxies cached 1606 * with 5 different node managers. These connections for these proxies will 1607 * be timed out if idle for more than the system wide idle timeout period. 1608 * Note that this could cause issues on large clusters as many connections 1609 * could linger simultaneously and lead to a large number of connection 1610 * threads. The token used for authentication will be used only at 1611 * connection creation time. If a new token is received then the earlier 1612 * connection should be closed in order to use the new token. This and 1613 * {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related 1614 * and should be in sync (no need for them to be equal). 1615 * If the value of this property is zero then the connection cache is 1616 * disabled and connections will use a zero idle timeout to prevent too 1617 * many connection threads on large clusters. 1618 */ 1619 public static final String NM_CLIENT_MAX_NM_PROXIES = 1620 YARN_PREFIX + "client.max-cached-nodemanagers-proxies"; 1621 public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 0; 1622 1623 /** Max time to wait to establish a connection to NM */ 1624 public static final String CLIENT_NM_CONNECT_MAX_WAIT_MS = 1625 YARN_PREFIX + "client.nodemanager-connect.max-wait-ms"; 1626 public static final long DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS = 1627 3 * 60 * 1000; 1628 1629 /** Time interval between each attempt to connect to NM */ 1630 public static final String CLIENT_NM_CONNECT_RETRY_INTERVAL_MS = 1631 YARN_PREFIX + "client.nodemanager-connect.retry-interval-ms"; 1632 public static final long DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS 1633 = 10 * 1000; 1634 1635 public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; 1636 public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY 1637 .name(); 1638 1639 1640 /** 1641 * Max time to wait for NM to connection to RM. 1642 * When not set, proxy will fall back to use value of 1643 * RESOURCEMANAGER_CONNECT_MAX_WAIT_MS. 1644 */ 1645 public static final String NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1646 YARN_PREFIX + "nodemanager.resourcemanager.connect.max-wait.ms"; 1647 1648 /** 1649 * Time interval between each NM attempt to connection to RM. 1650 * When not set, proxy will fall back to use value of 1651 * RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS. 1652 */ 1653 public static final String NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = 1654 YARN_PREFIX + "nodemanager.resourcemanager.connect.retry-interval.ms"; 1655 1656 /** 1657 * Node-labels configurations 1658 */ 1659 public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels."; 1660 1661 /** 1662 * Class for RMNodeLabelsManager Please note this value should be consistent 1663 * in client nodes and RM node(s) 1664 */ 1665 public static final String RM_NODE_LABELS_MANAGER_CLASS = NODE_LABELS_PREFIX 1666 + "manager-class"; 1667 1668 /** URI for NodeLabelManager */ 1669 public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX 1670 + "fs-store.root-dir"; 1671 public static final String FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = 1672 NODE_LABELS_PREFIX + "fs-store.retry-policy-spec"; 1673 public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = 1674 "2000, 500"; 1675 1676 public static final String AM_BLACKLISTING_ENABLED = 1677 YARN_PREFIX + "am.blacklisting.enabled"; 1678 public static final boolean DEFAULT_AM_BLACKLISTING_ENABLED = true; 1679 1680 public static final String AM_BLACKLISTING_DISABLE_THRESHOLD = 1681 YARN_PREFIX + "am.blacklisting.disable-failure-threshold"; 1682 public static final float DEFAULT_AM_BLACKLISTING_DISABLE_THRESHOLD = 0.8f; 1683 1684 public static final String APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC = 1685 YARN_PREFIX + "app.attempt.diagnostics.limit.kc"; 1686 1687 public static final int DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC = 64; 1688 1689 1690 public YarnConfiguration() { 1691 super(); 1692 } 1693 1694 public YarnConfiguration(Configuration conf) { 1695 super(conf); 1696 if (! (conf instanceof YarnConfiguration)) { 1697 this.reloadConfiguration(); 1698 } 1699 } 1700 1701 @Private 1702 public static List<String> getServiceAddressConfKeys(Configuration conf) { 1703 return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS 1704 : RM_SERVICES_ADDRESS_CONF_KEYS_HTTP; 1705 } 1706 1707 /** 1708 * Get the socket address for <code>name</code> property as a 1709 * <code>InetSocketAddress</code>. On a HA cluster, 1710 * this fetches the address corresponding to the RM identified by 1711 * {@link #RM_HA_ID}. 1712 * @param name property name. 1713 * @param defaultAddress the default value 1714 * @param defaultPort the default port 1715 * @return InetSocketAddress 1716 */ 1717 @Override 1718 public InetSocketAddress getSocketAddr( 1719 String name, String defaultAddress, int defaultPort) { 1720 String address; 1721 if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) { 1722 address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); 1723 } else { 1724 address = get(name, defaultAddress); 1725 } 1726 return NetUtils.createSocketAddr(address, defaultPort, name); 1727 } 1728 1729 @Override 1730 public InetSocketAddress updateConnectAddr(String name, 1731 InetSocketAddress addr) { 1732 String prefix = name; 1733 if (HAUtil.isHAEnabled(this)) { 1734 prefix = HAUtil.addSuffix(prefix, HAUtil.getRMHAId(this)); 1735 } 1736 return super.updateConnectAddr(prefix, addr); 1737 } 1738 1739 @Private 1740 public static int getRMDefaultPortNumber(String addressPrefix, 1741 Configuration conf) { 1742 if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) { 1743 return YarnConfiguration.DEFAULT_RM_PORT; 1744 } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) { 1745 return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT; 1746 } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) { 1747 return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT; 1748 } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) { 1749 return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT; 1750 } else if (addressPrefix 1751 .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) { 1752 return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT; 1753 } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) { 1754 return YarnConfiguration.DEFAULT_RM_ADMIN_PORT; 1755 } else { 1756 throw new HadoopIllegalArgumentException( 1757 "Invalid RM RPC address Prefix: " + addressPrefix 1758 + ". The valid value should be one of " 1759 + getServiceAddressConfKeys(conf)); 1760 } 1761 } 1762 1763 public static boolean useHttps(Configuration conf) { 1764 return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf 1765 .get(YARN_HTTP_POLICY_KEY, 1766 YARN_HTTP_POLICY_DEFAULT)); 1767 } 1768 1769 public static boolean shouldRMFailFast(Configuration conf) { 1770 return conf.getBoolean(YarnConfiguration.RM_FAIL_FAST, 1771 conf.getBoolean(YarnConfiguration.YARN_FAIL_FAST, 1772 YarnConfiguration.DEFAULT_YARN_FAIL_FAST)); 1773 } 1774 1775 @Private 1776 public static String getClusterId(Configuration conf) { 1777 String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID); 1778 if (clusterId == null) { 1779 throw new HadoopIllegalArgumentException("Configuration doesn't specify " + 1780 YarnConfiguration.RM_CLUSTER_ID); 1781 } 1782 return clusterId; 1783 } 1784}