DAOS-19005 test: debug daos_build failure

SConstruct

@@ -513,7 +513,8 @@ def scons():
     # also install to $PREFIX/lib to work with existing avocado test code
     if prereqs.test_requested():
         env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py'])
-        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"])
+        env.Install('$PREFIX/lib/daos/TESTING/ftest/',
+                    ['ftest.sh', "requirements-ftest.txt", "requirements-build.txt"])
 
     env.Install("$PREFIX/lib64/daos", "VERSION")
 

src/tests/ftest/dfuse/daos_build.py

@@ -19,7 +19,7 @@
 from run_utils import run_remote
 
 
 def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False):
     """Run an actual test from above."""
     # Create a pool, container and start dfuse.
     self.log_step('Creating a single pool and container')
@@ -101,21 +101,23 @@
 
     mount_dir = dfuse.mount_dir.value
     build_dir = os.path.join(mount_dir, 'daos')
+    venv_bin_path = os.path.dirname(sys.executable)
 
-    remote_env['PATH'] = f"{os.path.join(mount_dir, 'venv', 'bin')}:$PATH"
-    remote_env['VIRTUAL_ENV'] = os.path.join(mount_dir, 'venv')
+    remote_env['PATH'] = f"{venv_bin_path}:$PATH"
+    remote_env['VIRTUAL_ENV'] = os.path.dirname(venv_bin_path)
     remote_env['COVFILE'] = os.environ['COVFILE']
     remote_env['HTTPS_PROXY'] = os.environ.get('HTTPS_PROXY', '')
     remote_env['NO_PROXY'] = os.environ.get('NO_PROXY', '')
 
     if il_lib is not None:
+        il_lib_short = il_lib.rstrip('.so')
         remote_env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', il_lib)
-        remote_env['D_LOG_FILE'] = '/var/tmp/daos_testing/daos-il.log'
+        remote_env['D_LOG_FILE'] = f'/var/tmp/daos_testing/daos-{il_lib_short}.log'
         remote_env['DD_MASK'] = 'all'
         remote_env['DD_SUBSYS'] = 'all'
         remote_env['D_LOG_MASK'] = 'WARN,IL=WARN'
         if il_lib == 'libpil4dfs.so':
-            remote_env['D_IL_NO_BYPASS'] = '1'
+            # remote_env['D_IL_NO_BYPASS'] = '1'
             remote_env['D_IL_COMPATIBLE'] = '1'
             remote_env['D_IL_MAX_EQ'] = '0'
 
@@ -129,14 +131,11 @@
     elif "ubuntu" in distro_info.name.lower():
         distro = "ubuntu"
 
-    cmds = [f'{sys.executable} -m venv {mount_dir}/venv',
-            f'git clone https://github.com/daos-stack/daos.git {build_dir}',
+    cmds = [f'git clone https://github.com/daos-stack/daos.git {build_dir}',
             f'git -C {build_dir} checkout {__get_daos_build_checkout(self)}',
             f'git -C {build_dir} submodule update --init --recursive',
             f'cp {build_dir}/utils/scripts/install-{distro}.sh /tmp/install.sh',
             'sudo -E NO_OPENMPI_DEVEL=1 /tmp/install.sh -y',
-            'python3 -m pip install pip --upgrade',
-            f'python3 -m pip install -r {build_dir}/requirements-build.txt',
             f'scons -C {build_dir} --jobs {build_jobs} --build-deps=only',
             f'daos filesystem query {mount_dir}',
             f'daos filesystem evict {build_dir}',

src/tests/ftest/scripts/setup_nodes.sh

@@ -106,11 +106,12 @@
 [global]
     progress_bar = off
     no_color = true
-    quiet = 1
+    quiet = 0
 EOF
 
 pip install --upgrade pip
 pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-ftest.txt
+pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-build.txt
 # Copy the pydaos source locally and install it, in an ideal world this would install
 # from the read-only tree directly but for now that isn't working.
 # https://github.com/pypa/setuptools/issues/3237