NGSOK-1875 Add Hadoop group mapping provider for ACL group resolution

core/src/main/scala/org/apache/spark/security/HadoopGroupsMappingProvider.scala

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.security
+
+import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
+
+import org.apache.hadoop.security.Groups
+
+import org.apache.spark.SparkConf
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.internal.Logging
+
+/**
+ * This class is responsible for getting the groups for a particular user via the Hadoop group
+ * mapping service configured by `hadoop.security.group.mapping`, including composite or LDAP
+ * based mappings configured via `hadoop.security.group.mapping.providers`, so that Spark ACLs
+ * resolve groups from the same source as HDFS and YARN. The Hadoop configuration is loaded from
+ * the usual sources (e.g. `HADOOP_CONF_DIR` entries on the classpath) and can be extended or
+ * overridden with `spark.hadoop.*` properties. Lookups are cached by the Hadoop `Groups` service
+ * according to `hadoop.security.groups.cache.secs`.
+ */
+private[spark] class HadoopGroupsMappingProvider extends GroupMappingServiceProvider
+  with Logging {
+
+  // A dedicated Groups instance (rather than Groups.getUserToGroupsMappingService) so that the
+  // mapping honors this process' SparkConf overrides regardless of whether the shared service
+  // was already initialized elsewhere (e.g. by UserGroupInformation) with a different config.
+  private lazy val groups = new Groups(SparkHadoopUtil.get.newConfiguration(new SparkConf()))
+
+  override def getGroups(username: String): Set[String] = {
+    val userGroups = try {
+      groups.getGroupsSet(username).asScala.toSet
+    } catch {
+      // Groups throws IOException when the user is unknown or has no groups
+      case NonFatal(e) =>
+        logDebug(s"Unable to resolve groups for user: $username", e)
+        Set.empty[String]
+    }
+    logDebug("User: " + username + " Groups: " + userGroups.mkString(","))
+    userGroups
+  }
+}

core/src/test/scala/org/apache/spark/security/HadoopGroupsMappingProviderSuite.scala

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.security
+
+import java.util.{ArrayList => JArrayList, Arrays, List => JList}
+
+import org.apache.spark.SparkFunSuite
+
+class HadoopGroupsMappingProviderSuite extends SparkFunSuite {
+
+  private val mappingKey = "spark.hadoop.hadoop.security.group.mapping"
+
+  private def withTestGroupMapping(f: => Unit): Unit = {
+    System.setProperty(mappingKey, classOf[TestHadoopGroupsMapping].getName)
+    try {
+      f
+    } finally {
+      System.clearProperty(mappingKey)
+    }
+  }
+
+  test("resolves groups via the configured Hadoop group mapping service") {
+    withTestGroupMapping {
+      val provider = new HadoopGroupsMappingProvider()
+      assert(provider.getGroups("alice") === Set("wheel", "analysts"))
+    }
+  }
+
+  test("returns an empty set for users without groups") {
+    withTestGroupMapping {
+      val provider = new HadoopGroupsMappingProvider()
+      assert(provider.getGroups("bob") === Set.empty)
+    }
+  }
+}
+
+/**
+ * A deterministic Hadoop group mapping used instead of the OS/LDAP backed defaults.
+ * Must be a public top-level class so that Hadoop can instantiate it reflectively.
+ */
+class TestHadoopGroupsMapping extends org.apache.hadoop.security.GroupMappingServiceProvider {
+
+  override def getGroups(user: String): JList[String] = {
+    if (user == "alice") {
+      Arrays.asList("wheel", "analysts")
+    } else {
+      new JArrayList[String]()
+    }
+  }
+
+  override def cacheGroupsRefresh(): Unit = {}
+
+  override def cacheGroupsAdd(groups: JList[String]): Unit = {}
+}

docs/security.md

@@ -397,6 +397,13 @@ The following options control the authentication of Web UIs:
     <br /><em>Note:</em> This implementation supports only Unix/Linux-based environments.
     Windows environment is currently <b>not</b> supported. However, a new platform/protocol can
     be supported by implementing the trait mentioned above.
+
+    <br />Alternatively, <code>org.apache.spark.security.HadoopGroupsMappingProvider</code>
+    resolves groups through the Hadoop group mapping service configured by
+    <code>hadoop.security.group.mapping</code> (including composite or LDAP based mappings
+    configured via <code>hadoop.security.group.mapping.providers</code>), so that Spark ACLs
+    use the same group source as HDFS and YARN. The Hadoop properties can be set or overridden
+    with <code>spark.hadoop.*</code> entries in the Spark configuration.
   </td>
   <td>2.0.0</td>
 </tr>