From 82bb49e1eab4bc6e9dc55dcd73a522f749294716 Mon Sep 17 00:00:00 2001 From: Petr Fedchenkov Date: Wed, 8 Jul 2026 13:44:42 +0300 Subject: [PATCH] NGSOK-1875 Add Hadoop group mapping provider for ACL group resolution Add org.apache.spark.security.HadoopGroupsMappingProvider, an implementation of GroupMappingServiceProvider that delegates to the Hadoop Groups service. --- .../HadoopGroupsMappingProvider.scala | 58 ++++++++++++++++ .../HadoopGroupsMappingProviderSuite.scala | 69 +++++++++++++++++++ docs/security.md | 7 ++ 3 files changed, 134 insertions(+) create mode 100644 core/src/main/scala/org/apache/spark/security/HadoopGroupsMappingProvider.scala create mode 100644 core/src/test/scala/org/apache/spark/security/HadoopGroupsMappingProviderSuite.scala diff --git a/core/src/main/scala/org/apache/spark/security/HadoopGroupsMappingProvider.scala b/core/src/main/scala/org/apache/spark/security/HadoopGroupsMappingProvider.scala new file mode 100644 index 0000000000000..3300dad203ee8 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/security/HadoopGroupsMappingProvider.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.security + +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +import org.apache.hadoop.security.Groups + +import org.apache.spark.SparkConf +import org.apache.spark.deploy.SparkHadoopUtil +import org.apache.spark.internal.Logging + +/** + * This class is responsible for getting the groups for a particular user via the Hadoop group + * mapping service configured by `hadoop.security.group.mapping`, including composite or LDAP + * based mappings configured via `hadoop.security.group.mapping.providers`, so that Spark ACLs + * resolve groups from the same source as HDFS and YARN. The Hadoop configuration is loaded from + * the usual sources (e.g. `HADOOP_CONF_DIR` entries on the classpath) and can be extended or + * overridden with `spark.hadoop.*` properties. Lookups are cached by the Hadoop `Groups` service + * according to `hadoop.security.groups.cache.secs`. + */ +private[spark] class HadoopGroupsMappingProvider extends GroupMappingServiceProvider + with Logging { + + // A dedicated Groups instance (rather than Groups.getUserToGroupsMappingService) so that the + // mapping honors this process' SparkConf overrides regardless of whether the shared service + // was already initialized elsewhere (e.g. by UserGroupInformation) with a different config. + private lazy val groups = new Groups(SparkHadoopUtil.get.newConfiguration(new SparkConf())) + + override def getGroups(username: String): Set[String] = { + val userGroups = try { + groups.getGroupsSet(username).asScala.toSet + } catch { + // Groups throws IOException when the user is unknown or has no groups + case NonFatal(e) => + logDebug(s"Unable to resolve groups for user: $username", e) + Set.empty[String] + } + logDebug("User: " + username + " Groups: " + userGroups.mkString(",")) + userGroups + } +} diff --git a/core/src/test/scala/org/apache/spark/security/HadoopGroupsMappingProviderSuite.scala b/core/src/test/scala/org/apache/spark/security/HadoopGroupsMappingProviderSuite.scala new file mode 100644 index 0000000000000..46a9b1d7037ee --- /dev/null +++ b/core/src/test/scala/org/apache/spark/security/HadoopGroupsMappingProviderSuite.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.security + +import java.util.{ArrayList => JArrayList, Arrays, List => JList} + +import org.apache.spark.SparkFunSuite + +class HadoopGroupsMappingProviderSuite extends SparkFunSuite { + + private val mappingKey = "spark.hadoop.hadoop.security.group.mapping" + + private def withTestGroupMapping(f: => Unit): Unit = { + System.setProperty(mappingKey, classOf[TestHadoopGroupsMapping].getName) + try { + f + } finally { + System.clearProperty(mappingKey) + } + } + + test("resolves groups via the configured Hadoop group mapping service") { + withTestGroupMapping { + val provider = new HadoopGroupsMappingProvider() + assert(provider.getGroups("alice") === Set("wheel", "analysts")) + } + } + + test("returns an empty set for users without groups") { + withTestGroupMapping { + val provider = new HadoopGroupsMappingProvider() + assert(provider.getGroups("bob") === Set.empty) + } + } +} + +/** + * A deterministic Hadoop group mapping used instead of the OS/LDAP backed defaults. + * Must be a public top-level class so that Hadoop can instantiate it reflectively. + */ +class TestHadoopGroupsMapping extends org.apache.hadoop.security.GroupMappingServiceProvider { + + override def getGroups(user: String): JList[String] = { + if (user == "alice") { + Arrays.asList("wheel", "analysts") + } else { + new JArrayList[String]() + } + } + + override def cacheGroupsRefresh(): Unit = {} + + override def cacheGroupsAdd(groups: JList[String]): Unit = {} +} diff --git a/docs/security.md b/docs/security.md index b3743d9a1b0a6..77530b2c50007 100644 --- a/docs/security.md +++ b/docs/security.md @@ -434,6 +434,13 @@ The following options control the authentication of Web UIs:
Note: This implementation supports only Unix/Linux-based environments. Windows environment is currently not supported. However, a new platform/protocol can be supported by implementing the trait mentioned above. + +
Alternatively, org.apache.spark.security.HadoopGroupsMappingProvider + resolves groups through the Hadoop group mapping service configured by + hadoop.security.group.mapping (including composite or LDAP based mappings + configured via hadoop.security.group.mapping.providers), so that Spark ACLs + use the same group source as HDFS and YARN. The Hadoop properties can be set or overridden + with spark.hadoop.* entries in the Spark configuration. 2.0.0