DAOS-19523 test: use runner as client

ftest.sh

@@ -22,8 +22,10 @@ NFS_SERVER=${NFS_SERVER:-${HOSTNAME%%.*}}
 
 trap 'echo "encountered an unchecked return code, exiting with error"' ERR
 
+# Use localhost only as a client
 IFS=" " read -r -a nodes <<< "${2//,/ }"
-TEST_NODES=$(IFS=","; echo "${nodes[*]:1}")
+TEST_SERVER_NODES=$(IFS=","; echo "${nodes[*]:1}")
+TEST_CLIENT_NODES=$(IFS=","; echo "${nodes[*]}")
 
 # Optional arguments for launch.py
 LAUNCH_OPT_ARGS="${3:-}"
@@ -122,15 +124,15 @@ _DAOS_NO_PROXY=${DAOS_NO_PROXY:-}
 # shellcheck disable=SC2029
 # shellcheck disable=SC2086
 if ! ssh -A $SSH_KEY_ARGS ${REMOTE_ACCT:-jenkins}@"${nodes[0]}" \
-    "FIRST_NODE=\"${nodes[0]}\"
-     TEST_RPMS=\"$TEST_RPMS\"
+    "TEST_RPMS=\"$TEST_RPMS\"
      DAOS_TEST_SHARED_DIR=\"${DAOS_TEST_SHARED_DIR:-$PWD/install/tmp}\"
      DAOS_BASE=\"$DAOS_BASE\"
      TEST_TAG_DIR=\"$TEST_TAG_DIR\"
      PREFIX=\"$PREFIX\"
      SETUP_ONLY=\"${SETUP_ONLY:-false}\"
      TEST_TAG_ARG=\"$TEST_TAG_ARG\"
-     TEST_NODES=\"$TEST_NODES\"
+     TEST_SERVER_NODES=\"$TEST_SERVER_NODES\"
+     TEST_CLIENT_NODES=\"$TEST_CLIENT_NODES\"
      LAUNCH_OPT_ARGS=\"$LAUNCH_OPT_ARGS\"
      WITH_VALGRIND=\"$WITH_VALGRIND\"
      STAGE_NAME=\"$STAGE_NAME\"

src/tests/ftest/daos_test/nvme_recovery.yaml

@@ -2,6 +2,7 @@
 # required quantity is indicated by the placeholders
 hosts:
   test_servers: 2
+  test_clients: 1
 
 timeout: 600
 

src/tests/ftest/daos_test/rebuild.yaml

@@ -2,6 +2,7 @@
 # required quantity is indicated by the placeholders
 hosts:
   test_servers: 4
+  test_clients: 1
 
 timeout: 800
 timeouts:

src/tests/ftest/daos_test/suite.yaml

@@ -2,6 +2,7 @@
 # required quantity is indicated by the placeholders
 hosts:
   test_servers: 4
+  test_clients: 1
 
 # Note that subtests below can set their own timeout so this
 # should be a general average of all tests not including outliers

src/tests/ftest/io/unaligned_io.yaml

@@ -1,5 +1,6 @@
 hosts:
   test_servers: 4
+  test_clients: 1
 
 timeout: 900
 

src/tests/ftest/launch.py

@@ -359,7 +359,8 @@ def _run(self, args):
         try:
             group.update_test_yaml(
                 logger, args.scm_size, args.scm_mount, args.extra_yaml,
-                args.timeout_multiplier, args.override, args.verbose, args.include_localhost)
+                args.timeout_multiplier, args.override, args.verbose, args.include_localhost,
+                args.shared_server_client)
         except (RunException, YamlException) as e:
             message = f"Error modifying the test yaml files: {e}"
             status |= self.get_exit_status(1, message, "Setup", sys.exc_info())
@@ -568,6 +569,10 @@ def main():
         "-i", "--include_localhost",
         action="store_true",
         help="include the local host when cleaning and archiving")
+    parser.add_argument(
+        "-ssc", "--shared_server_client",
+        action="store_true",
+        help="potentially use the same nodes for both servers and clients")
     parser.add_argument(
         "-ins", "--insecure_mode",
         action="store_true",

src/tests/ftest/rebuild/inc_reint.yaml

@@ -1,5 +1,6 @@
 hosts:
   test_servers: 4
+  test_clients: 1
 
 timeout: 1H
 

src/tests/ftest/recovery/cat_recov_core.yaml

@@ -1,5 +1,6 @@
 hosts:
   test_servers: 4
+  test_clients: 1
 
 timeout: 1H40M
 

src/tests/ftest/scripts/main.sh

@@ -88,12 +88,12 @@ if [ -n "$DAOS_NO_PROXY" ]; then
     export NO_PROXY="${DAOS_NO_PROXY:-""}"
 fi
 
-launch_node_args="-ts ${TEST_NODES}"
+launch_node_args="-ts ${TEST_SERVER_NODES} -tc ${TEST_CLIENT_NODES}"
 if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then
     # Currently the 'Functional Hardware 24' uses a cluster that has 8 hosts configured to run
     # daos engines and the remaining hosts are configured to be clients. Use separate -ts and -tc
     # launch.py arguments to ensure these hosts are not used for unintended role
-    IFS=" " read -r -a test_node_list <<< "${TEST_NODES//,/ }"
+    IFS=" " read -r -a test_node_list <<< "${TEST_SERVER_NODES//,/ }"
     server_nodes=$(IFS=','; echo "${test_node_list[*]:0:8}")
     client_nodes=$(IFS=','; echo "${test_node_list[*]:8}")
     launch_node_args="-ts ${server_nodes} -tc ${client_nodes}"

src/tests/ftest/util/daos_core_base.py

@@ -26,7 +26,6 @@ def __init__(self, *args, **kwargs):
         """Initialize the DaosCoreBase object."""
         super().__init__(*args, **kwargs)
         self.subtest_name = None
-        self.using_local_host = False
 
     def setUp(self):
         """Set up before each test."""

src/tests/ftest/util/launch_utils.py

@@ -1023,7 +1023,7 @@ def list_tests(self, logger, verbose):
                             file_name, class_name, method_name, ','.join(tags))
 
     def update_test_yaml(self, logger, scm_size, scm_mount, extra_yaml, multiplier, override,
-                         verbose, include_localhost):
+                         verbose, include_localhost, shared_server_client=False):
         """Update each test yaml file.
 
         Args:
@@ -1035,6 +1035,8 @@ def update_test_yaml(self, logger, scm_size, scm_mount, extra_yaml, multiplier,
             override (bool): whether or not to override the number of hosts for the test
             verbose (int): level of verbosity
             include_localhost (bool): whether or not to include the local host with the client hosts
+            shared_server_client (bool, optional): whether the test yaml potentially uses the same
+                nodes for both servers and clients. Defaults to False.
 
         Raises:
             RunException: if there is an error modifying the test yaml files
@@ -1078,7 +1080,10 @@ def update_test_yaml(self, logger, scm_size, scm_mount, extra_yaml, multiplier,
         if extra_yaml:
             logger.debug("Updating placeholders in extra yaml files: %s", extra_yaml)
             common_extra_yaml = [
-                updater.update(extra, self._yaml_directory) or extra for extra in extra_yaml]
+                updater.update(
+                    extra, self._yaml_directory,
+                    shared_server_client=shared_server_client) or extra
+                for extra in extra_yaml]
             for test in self.tests:
                 test.extra_yaml.extend(common_extra_yaml)
 

src/tests/ftest/util/run_utils.py

@@ -465,6 +465,10 @@ def run_remote(log, hosts, command, verbose=True, timeout=120, task_debug=False,
     Returns:
         CommandResult: groups of command results from the same hosts with the same return status
     """
+    if hosts is None:
+        raise ValueError("No hosts specified for run_remote()")
+    if command is None:
+        raise ValueError("No command specified for run_remote()")
     task = task_self()
     task.set_info('debug', task_debug)
     task.set_default("stderr", stderr)

src/tests/ftest/util/yaml_utils.py

@@ -14,8 +14,10 @@
 # pylint: disable=import-error,no-name-in-module
 try:
     from util.data_utils import dict_extract_values, list_flatten, list_unique
+    from util.host_utils import get_local_host
 except (ImportError, ModuleNotFoundError):
     from data_utils import dict_extract_values, list_flatten, list_unique
+    from host_utils import get_local_host
 
 
 class YamlException(BaseException):
@@ -128,6 +130,7 @@ def __init__(self, logger, servers, clients, storage, timeout, override, verbose
             clients (NodeSet): set of hosts to use as clients in the test yaml
             storage (str): updated storage information to apply to the test yaml
             timeout (int): multiplier to apply to any timeouts specified in the test yaml
+            override (bool): whether to override the test yaml values with user-specified values
             verbose (int): verbosity level
         """
         self.log = logger
@@ -147,12 +150,14 @@ def placeholder_updates(self):
         """
         return self._servers or self._storage or self._timeout
 
-    def update(self, yaml_file, yaml_dir):
+    def update(self, yaml_file, yaml_dir, shared_server_client=False):
         """Update the placeholders in the specified test yaml file.
 
         Args:
             yaml_file (str): test yaml file to update
             yaml_dir (str): directory in which to write the updated test yaml file
+            shared_server_client (bool, optional): whether the test yaml potentially uses the same
+                nodes for both servers and clients. Defaults to False.
 
         Raises:
             YamlException: if any placeholders are found without a replacement
@@ -162,14 +167,16 @@ def update(self, yaml_file, yaml_dir):
 
         """
         self.log.debug("-" * 80)
-        replacements = self.get_replacements(yaml_file)
+        replacements = self.get_replacements(yaml_file, shared_server_client=shared_server_client)
         return self.apply_changes(yaml_file, yaml_dir, replacements)
 
-    def get_replacements(self, yaml_file):
+    def get_replacements(self, yaml_file, shared_server_client=False):
         """Determine the replacement values for the placeholders in the test yaml.
 
         Args:
             yaml_file (str): the test yaml file
+            shared_server_client (bool, optional): whether the test yaml potentially uses the same
+                nodes for both servers and clients. Defaults to False.
 
         Raises:
             YamlException: if there was a problem replacing any of the placeholders
@@ -178,9 +185,9 @@ def get_replacements(self, yaml_file):
             dict: a dictionary of existing test yaml entry keys and their replacement values
 
         """
-        replacements = {}
+        chosen_replacements = {}
         if not self.placeholder_updates:
-            return replacements
+            return chosen_replacements
 
         # Find the test yaml keys and values that match the replaceable fields
         yaml_data = get_yaml_data(yaml_file)
@@ -194,37 +201,57 @@ def get_replacements(self, yaml_file):
                 placeholder_data[key] = values if len(values) > 1 else values[0]
 
         # Generate a list of values that can be used as replacements
-        replacement_data = OrderedDict()
+        candidate_replacement_data = OrderedDict()
         for key, attr_name, _ in self.YAML_KEYS:
             args_value = getattr(self, attr_name)
             if isinstance(args_value, NodeSet):
-                replacement_data[key] = list(args_value)
+                candidate_replacement_data[key] = list(args_value)
             elif isinstance(args_value, str):
-                replacement_data[key] = args_value.split(",")
+                candidate_replacement_data[key] = args_value.split(",")
             elif args_value:
-                replacement_data[key] = [args_value]
+                candidate_replacement_data[key] = [args_value]
             else:
-                replacement_data[key] = None
+                candidate_replacement_data[key] = None
 
         # Assign replacement values for the test yaml entries to be replaced
         placeholder_keys = [yaml_key[0] for yaml_key in self.YAML_KEYS]
         self.log.debug("Detecting replacements for %s in %s", placeholder_keys, yaml_file)
         self.log.debug("  Placeholder data: %s", placeholder_data)
-        self.log.debug("  Replacement data:  %s", dict(replacement_data))
+        self.log.debug("  Replacement data:  %s", dict(candidate_replacement_data))
 
         node_mapping = {}
-        for key, replacement in replacement_data.items():
-            # If the user did not provide a specific list of replacement test_clients values, use
-            # the remaining test_servers values to replace test_clients placeholder values
-            if key == "test_clients" and not replacement:
-                replacement = replacement_data["test_servers"]
-
+        for key, candidate_replacements in candidate_replacement_data.items():
             if key not in placeholder_data:
                 if self._verbose > 1:
                     self.log.debug("  - No '%s' placeholder specified in the test yaml", key)
                 continue
 
-            if not replacement:
+            # If the user did not provide a specific list of replacement test_clients values, use
+            # the remaining test_servers values to replace test_clients placeholder values
+            if key == "test_clients" and not candidate_replacements:
+                candidate_replacements = candidate_replacement_data["test_servers"]
+
+            # If not sharing servers and clients, remove chosen servers from the candidate clients.
+            # If sharing, we still prefer exclusive if possible,
+            # so sort the candidate clients so that the chosen servers are last in the list.
+            if key == "test_clients":
+                if not shared_server_client:
+                    candidate_replacements = list(
+                        set(candidate_replacements)
+                        - set(chosen_replacements.get("test_servers", [])))
+                else:
+                    candidate_replacements.sort(
+                        key=lambda x: x in chosen_replacements.get("test_servers", []))
+
+            # localhost is preferred as a client over a server,
+            # so put localhost last for servers and first for clients.
+            # It will only be used as a server if it is the last available node.
+            if key == "test_servers":
+                candidate_replacements.sort(key=lambda x: x == str(get_local_host()))
+            if key == "test_clients":
+                candidate_replacements.sort(key=lambda x: x != str(get_local_host()))
+
+            if not candidate_replacements:
                 if self._verbose > 1:
                     self.log.debug(
                         "  - No replacement value for the '%s' placeholder: %s",
@@ -234,31 +261,34 @@ def get_replacements(self, yaml_file):
             # Replace test yaml keys that were:
             #   - found in the test yaml
             #   - have a user-specified replacement
-            if key.startswith("test_"):
+            if key in ("test_servers", "test_clients"):
                 # The entire server/client test yaml list entry is replaced by a new test yaml list
                 # entry, e.g.
                 #   '  test_servers: server-[1-2]' --> '  test_servers: wolf-[10-11]'
                 #   '  test_servers: 4'            --> '  test_servers: wolf-[10-13]'
                 self._get_host_replacement(
-                    replacements, placeholder_data, key, replacement, node_mapping)
+                    chosen_replacements, placeholder_data, key, candidate_replacements,
+                    node_mapping)
 
             elif key == "bdev_list":
                 # Individual bdev_list NVMe PCI addresses in the test yaml file are replaced with
                 # the new NVMe PCI addresses in the order they are found, e.g.
                 #   0000:81:00.0 --> 0000:12:00.0
-                self._get_storage_replacement(replacements, placeholder_data, key, replacement)
+                self._get_storage_replacement(chosen_replacements, placeholder_data, key,
+                                              candidate_replacements)
 
             else:
                 # Timeouts - replace the entire timeout entry (key + value) with the same key with
                 # its original value multiplied by the user-specified value, e.g.
                 #   timeout: 60 -> timeout: 600
-                self._get_timeout_replacement(replacements, placeholder_data, key, replacement)
+                self._get_timeout_replacement(chosen_replacements, placeholder_data, key,
+                                              candidate_replacements)
 
         # Display the replacement values
-        for value, replacement in list(replacements.items()):
-            self.log.debug("  - Replacement: %s -> %s", value, replacement)
+        for value, chosen_replacement in list(chosen_replacements.items()):
+            self.log.debug("  - Replacement: %s -> %s", value, chosen_replacement)
 
-        return replacements
+        return chosen_replacements
 
     def _get_host_replacement(self, replacements, placeholder_data, key, replacement, node_mapping):
         """Replace the server or client placeholders.