diff --git a/Jenkinsfile b/Jenkinsfile index c5ace14abd7..0a0bc3cdd1f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -480,6 +480,36 @@ String sconsArgs() { return sconsFaultsArgs() + ' ' + params.CI_SCONS_ARGS } +// This is a temporary workaround for a bug in pipeline-lib proxy handling. +// Because we can not risk failures until we verify that that bug can be +// fixed, we will use this function to remove the bug for each active branch +// Proxy rules: +// IF jenkins system has an HTTPS_PROXY set then it needs to be passed +// to the docker build command. If the system does not have an HTTPS_PROXY +// set then the docker build MUST NOT include an HTTPS_PROXY build argument. +// +// The DAOS_NO_PROXY build argument was put in because the HTTPS_PROXY +// build argument was being passed to the docker build when it should not +// have been instead of fixing the bug in pipeline-lib. +// This function will be removed when the bug is fixed. +// The use of no_proxy/NO_PROXY environment variables to avoid a proxy is +// unsafe because there is no single standard used for for what is legal to +// be in it. +// +// If a local mirror is available via REPOSITORY_URL/ARTIFACTORY_URL it +// should always be used instead of a proxy server for reliability. +// Applications should where possible always use configuration file to +// for any needed proxy or mirror settings instead of environment variables +// which will avoid needing no_proxy issues. +String daosDockerBuildArgs(Map args = [:]) { + String buildArgs = dockerBuildArgs(args) + if (!env.HTTPS_PROXY) { + buildArgs += ' --build-arg HTTPS_PROXY=' + buildArgs += ' --build-arg DAOS_NO_PROXY=' + } + return buildArgs +} + /** * Update default commit pragmas based on files modified. */ @@ -503,7 +533,6 @@ pipeline { TEST_RPMS = cachedCommitPragma(pragma: 'RPM-test', def_val: 'true') COVFN_DISABLED = cachedCommitPragma(pragma: 'Skip-fnbullseye', def_val: 'true') REPO_FILE_URL = repoFileUrl(env.REPO_FILE_URL) - HTTPS_PROXY = '' PYTHON_VERSION = '3.11' } @@ -614,14 +643,19 @@ pipeline { booleanParam(name: bashName('Functional on EL 9 with Valgrind'), defaultValue: false, description: 'Run the Functional on EL 9 with Valgrind stage.') + // >>> TEMP PATCH: shift default functional testing from EL 9 to Leap 15/SLES 15 <<< + // >>> REVERT these 3 defaultValue lines back to EL9=true, Leap15=false, SLES15=false <<< booleanParam(name: bashName('Functional on EL 9'), - defaultValue: true, + // defaultValue: true, + defaultValue: false, description: 'Run the Functional on EL 9 stage.') booleanParam(name: bashName('Functional on Leap 15'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional on Leap 15 stage.') booleanParam(name: bashName('Functional on SLES 15'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional on SLES 15 stage.') booleanParam(name: bashName('Functional on Ubuntu 20.04'), defaultValue: false, @@ -629,6 +663,7 @@ pipeline { booleanParam(name: bashName('Fault injection testing'), defaultValue: true, description: 'Run the Fault injection testing stage.') + // >>> END TEMP PATCH <<< booleanParam(name: bashName('Test RPMs on EL 9'), defaultValue: true, description: 'Run the Test RPMs on EL 9 stage.') @@ -638,26 +673,33 @@ pipeline { booleanParam(name: bashName('Test Hardware'), defaultValue: true, description: 'Run the Test Hardware stage.') + // >>> TEMP PATCH: run all hardware stages on SLES 15.7 for PR coverage <<< + // >>> REVERT these false->true defaultValue lines after SLES/Leap validation <<< booleanParam(name: bashName('Functional Hardware Medium'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional Hardware Medium stage.') booleanParam(name: bashName('Functional Hardware Medium MD on SSD'), defaultValue: true, description: 'Run the Functional Hardware Medium MD on SSD stage.') booleanParam(name: bashName('Functional Hardware Medium VMD'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional Hardware Medium VMD stage.') booleanParam(name: bashName('Functional Hardware Medium Verbs Provider'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional Hardware Medium Verbs Provider stage.') booleanParam(name: bashName('Functional Hardware Medium Verbs Provider MD on SSD'), defaultValue: true, description: 'Run the Functional Hardware Medium Verbs Provider MD on SSD stage.') booleanParam(name: bashName('Functional Hardware Medium UCX Provider'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional Hardware Medium UCX Provider stage.') booleanParam(name: bashName('Functional Hardware Large'), - defaultValue: false, + // defaultValue: false, + defaultValue: true, description: 'Run the Functional Hardware Large stage.') booleanParam(name: bashName('Functional Hardware Large MD on SSD'), defaultValue: true, @@ -668,6 +710,7 @@ pipeline { booleanParam(name: bashName('Functional Cluster Box Medium Verbs Provider MD on SSD'), defaultValue: true, description: 'Run the Functional Cluster Box Verbs Provider test stage') + // >>> END TEMP PATCH <<< string(name: 'CI_UNIT_VM1_LABEL', defaultValue: 'ci_vm1', description: 'Label to use for 1 VM node unit and RPM tests') @@ -833,7 +876,7 @@ pipeline { ' --build-arg REPOS="' + prRepos() + '"' + ' --build-arg POINT_RELEASE=.7' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + - " --build-arg DAOS_DEPS_INSTALL=yes" + ' --build-arg DAOS_DEPS_INSTALL=yes' } } steps { @@ -886,14 +929,14 @@ pipeline { dockerfile { filename 'utils/docker/Dockerfile.leap.15' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(repo_type: 'stable', - parallel_build: true, - deps_build: true) + + additionalBuildArgs daosDockerBuildArgs(repo_type: 'stable', + parallel_build: true, + deps_build: true) + " -t ${sanitized_JOB_NAME()}-leap15" + ' --target build-ci' + ' --build-arg POINT_RELEASE=.6' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + - " --build-arg DAOS_DEPS_INSTALL=yes" + ' --build-arg DAOS_DEPS_INSTALL=yes' } } steps { @@ -931,7 +974,8 @@ pipeline { stage('Unit Tests') { when { beforeAgent true - expression { shouldStageRun('Unit Tests') } + // TEMP PR-18944: skip unit tests to speed up diagnostic reruns. + expression { false && shouldStageRun('Unit Tests') } } parallel { stage('Unit Test') { @@ -1271,7 +1315,8 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Medium MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Medium MD on SSD', @@ -1283,7 +1328,8 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Medium VMD': getFunctionalTestStage( name: 'Functional Hardware Medium VMD', @@ -1296,7 +1342,8 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Medium Verbs Provider': getFunctionalTestStage( name: 'Functional Hardware Medium Verbs Provider', @@ -1309,7 +1356,8 @@ pipeline { default_nvme: 'auto', provider: 'ofi+verbs;ofi_rxm', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Medium Verbs Provider MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Medium Verbs Provider MD on SSD', @@ -1322,7 +1370,8 @@ pipeline { default_nvme: 'auto_md_on_ssd', provider: 'ofi+verbs;ofi_rxm', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Medium UCX Provider': getFunctionalTestStage( name: 'Functional Hardware Medium UCX Provider', @@ -1335,7 +1384,8 @@ pipeline { default_nvme: 'auto', provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'), job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Large': getFunctionalTestStage( name: 'Functional Hardware Large', @@ -1347,7 +1397,8 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Hardware Large MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Large MD on SSD', @@ -1359,7 +1410,8 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto_md_on_ssd', job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Cluster Box Medium MD on SSD': getFunctionalTestStage( name: 'Functional Cluster Box Medium MD on SSD', @@ -1374,7 +1426,8 @@ pipeline { run_if_pr: true, run_if_landing: false, job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), 'Functional Cluster Box Medium Verbs Provider MD on SSD': getFunctionalTestStage( name: 'Functional Cluster Box Medium Verbs Provider MD on SSD', @@ -1390,7 +1443,8 @@ pipeline { run_if_pr: true, run_if_landing: false, job_status: job_status_internal, - image_version: 'el9.7' + // image_version: 'el9.7' + image_version: 'sles15.7' ), ) } diff --git a/ci/provisioning/post_provision_config_nodes_LEAP.sh b/ci/provisioning/post_provision_config_nodes_LEAP.sh index a27d3dd6fa8..05427fdd2ae 100644 --- a/ci/provisioning/post_provision_config_nodes_LEAP.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP.sh @@ -9,7 +9,8 @@ bootstrap_dnf() { rm -rf "$REPOS_DIR" ln -s ../zypp/repos.d "$REPOS_DIR" dnf -y remove lua-lmod - dnf -y --nogpgcheck install lua-lmod --repo '*lua*' --repo '*network-cluster*' --repo '*oss-proxy*' + dnf -y --nogpgcheck install lua-lmod \ + --repo '*network-cluster*' --repo '*oss-proxy*' } group_repo_post() { diff --git a/ftest.sh b/ftest.sh index 2c7090a1a0d..49e0e5ac062 100755 --- a/ftest.sh +++ b/ftest.sh @@ -117,6 +117,7 @@ args="${1:-quick}" shift || true args+=" $*" +_HTTPS_PROXY=${HTTPS_PROXY:-} _DAOS_HTTPS_PROXY=${DAOS_HTTPS_PROXY:-} _DAOS_NO_PROXY=${DAOS_NO_PROXY:-} # shellcheck disable=SC2029 @@ -134,7 +135,9 @@ if ! ssh -A $SSH_KEY_ARGS ${REMOTE_ACCT:-jenkins}@"${nodes[0]}" \ LAUNCH_OPT_ARGS=\"$LAUNCH_OPT_ARGS\" WITH_VALGRIND=\"$WITH_VALGRIND\" STAGE_NAME=\"$STAGE_NAME\" - DAOS_HTTPS_PROXY=\"$_DAOS_HTTPS_PROXY\" + HTTPS_PROXY=\"$_HTTPS_PROXY\" + # TEMP DISABLED escape hatch: uncomment to force DAOS_HTTPS_PROXY through as a quick fix. + # DAOS_HTTPS_PROXY=\"$_DAOS_HTTPS_PROXY\" DAOS_NO_PROXY=\"$_DAOS_NO_PROXY\" DAOS_FTEST_VENV=\"$DAOS_FTEST_VENV\" $(sed -e '1,/^$/d' "$SCRIPT_LOC"/main.sh)"; then diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index f075d43c696..fba2933c02f 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -80,10 +80,13 @@ export TEST_RPMS export DAOS_BASE export DAOS_TEST_APP_SRC=${DAOS_TEST_APP_SRC:-"/CIShare/daos_test/apps"} export DAOS_TEST_APP_DIR=${DAOS_TEST_APP_DIR:-"${DAOS_TEST_SHARED_DIR}/daos_test/apps"} -if [ -n "$DAOS_HTTPS_PROXY" ]; then - # shellcheck disable=SC2154 - export HTTPS_PROXY="${DAOS_HTTPS_PROXY:-""}" -fi +# HTTPS_PROXY is passed in directly by ftest.sh. A high-level script like this +# should not be the one assigning HTTPS_PROXY=$DAOS_HTTPS_PROXY; revisit after +# smoke testing and document proper CI/lab proxy usage. +# if [ -n "$DAOS_HTTPS_PROXY" ]; then +# # shellcheck disable=SC2154 +# export HTTPS_PROXY="${DAOS_HTTPS_PROXY:-""}" +# fi if [ -n "$DAOS_NO_PROXY" ]; then export NO_PROXY="${DAOS_NO_PROXY:-""}" fi diff --git a/src/utils/daos_hdlr.c b/src/utils/daos_hdlr.c index 68f217cd12c..8170259f672 100644 --- a/src/utils/daos_hdlr.c +++ b/src/utils/daos_hdlr.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1314,8 +1314,11 @@ dm_serialize_cont_md(struct cmd_args_s *ap, struct dm_args *ca, daos_prop_t *pro } handle = dlopen(LIBSERIALIZE, RTLD_NOW); if (handle == NULL) { + const char *dl_err = dlerror(); + rc = -DER_INVAL; - DH_PERROR_DER(ap, rc, "libdaos_serialize.so not found"); + DH_PERROR_DER(ap, rc, "failed to load %s: %s", LIBSERIALIZE, + dl_err ? dl_err : "unknown dynamic loader error"); D_GOTO(out, rc); } daos_cont_serialize_md = dlsym(handle, "daos_cont_serialize_md"); @@ -1342,8 +1345,11 @@ dm_deserialize_cont_md(struct cmd_args_s *ap, struct dm_args *ca, char *preserve handle = dlopen(LIBSERIALIZE, RTLD_NOW); if (handle == NULL) { + const char *dl_err = dlerror(); + rc = -DER_INVAL; - DH_PERROR_DER(ap, rc, "libdaos_serialize.so not found"); + DH_PERROR_DER(ap, rc, "failed to load %s: %s", LIBSERIALIZE, + dl_err ? dl_err : "unknown dynamic loader error"); D_GOTO(out, rc); } daos_cont_deserialize_props = dlsym(handle, "daos_cont_deserialize_props"); @@ -1370,8 +1376,11 @@ dm_deserialize_cont_attrs(struct cmd_args_s *ap, struct dm_args *ca, char *prese handle = dlopen(LIBSERIALIZE, RTLD_NOW); if (handle == NULL) { + const char *dl_err = dlerror(); + rc = -DER_INVAL; - DH_PERROR_DER(ap, rc, "libdaos_serialize.so not found"); + DH_PERROR_DER(ap, rc, "failed to load %s: %s", LIBSERIALIZE, + dl_err ? dl_err : "unknown dynamic loader error"); D_GOTO(out, rc); } daos_cont_deserialize_attrs = dlsym(handle, "daos_cont_deserialize_attrs"); diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index daf60549386..1c2d2b4776d 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -31,26 +31,35 @@ ARG JENKINS_URL ARG REPOS ARG DAOS_LAB_CA_FILE_URL -# Accept DAOS_HTTP_PROXY, DAOS_HTTPS_PROXY, and DAOS_NO_PROXY at build time -ARG DAOS_HTTP_PROXY +# Accept DAOS_HTTPS_PROXY, and DAOS_NO_PROXY at build time +# DAOS_HTTPS_PROXY is used safely pass a PROXY to the build environment with +# out making everything use it, which can cause problems. +# DAOS_HTTPS_PROXY is used when you do not have a local artifact server +# and you need to use a proxy server that is not a smart proxy. +# +# HTTPS_PROXY is an automatic ARG, that will be global to the build environment +# if it is passed. If HTTPS_PROXY is passed, normally the DAOS_HTTPS_PROXY +# should not be passed. HTTPS_PROXY should be a smart proxy that can +# can pass through to URLs that resolve to local IP addresses. +# +# DAOS_NO_PROXY is used to safely pass a NO_PROXY list to the build +# environment without making everything use it, which can cause problems. +# The no_proxy/NO_PROXY environment variables are risky to use as there are +# conflicts in how they should be defined for different applications. ARG DAOS_HTTPS_PROXY ARG DAOS_NO_PROXY # Propagate into the build environment -ENV http_proxy=${DAOS_HTTP_PROXY} \ - HTTP_PROXY=${DAOS_HTTP_PROXY} \ - https_proxy=${DAOS_HTTPS_PROXY} \ - HTTPS_PROXY=${DAOS_HTTPS_PROXY} \ - no_proxy=${DAOS_NO_PROXY} \ +ENV no_proxy=${DAOS_NO_PROXY} \ NO_PROXY=${DAOS_NO_PROXY} # Persist into /etc/environment for use by shells and services RUN set -e; \ - if [ -n "$DAOS_HTTP_PROXY" ]; then \ - echo "http_proxy=$DAOS_HTTP_PROXY" >> /etc/environment; \ - echo "HTTP_PROXY=$DAOS_HTTP_PROXY" >> /etc/environment; \ - fi; \ if [ -n "$DAOS_HTTPS_PROXY" ]; then \ - echo "https_proxy=$DAOS_HTTPS_PROXY" >> /etc/environment; \ - echo "HTTPS_PROXY=$DAOS_HTTPS_PROXY" >> /etc/environment; \ + echo "daos_https_proxy=$DAOS_HTTPS_PROXY" >> /etc/environment; \ + echo "DAOS_HTTPS_PROXY=$DAOS_HTTPS_PROXY" >> /etc/environment; \ + fi; \ + if [ -n "$HTTPS_PROXY" ]; then \ + echo "https_proxy=$HTTPS_PROXY" >> /etc/environment; \ + echo "HTTPS_PROXY=$HTTPS_PROXY" >> /etc/environment; \ fi; \ if [ -n "$DAOS_NO_PROXY" ]; then \ echo "no_proxy=$DAOS_NO_PROXY" >> /etc/environment; \ @@ -75,29 +84,14 @@ RUN dnf remove -y $DNF_UPGRADE_EXCLUDES || true && \ # Setup docker image for gcc build only. This is done in a separate stage # to allow for a smaller image for CI building only. FROM basic AS build-ci +ARG JENKINS_URL # Install minimal set of packages for building DAOS with gcc compiler and # no Java support. RUN --mount=type=bind,source=utils/scripts/install-leap15.sh,target=/tmp/daos/install.sh \ - INSTALL_BUILD_CI_ONLY="true" /tmp/daos/install.sh && \ + JENKINS_URL="$JENKINS_URL" INSTALL_BUILD_CI_ONLY="true" /tmp/daos/install.sh && \ dnf clean all -# According to https://pkgs.org/search/?q=lua-lmod, Leap 15.6 only has lua-lmod-8.7.34. -# This version has a problem with loading modules, as described in https://github.com/TACC/Lmod/issues/687. -# This affects MPI detection in scons, so DAOS cannot be built with MPI support. -# A custom source of the package is required in CI to install a valid version -# of lua-lmod and its dependencies: -# opensuse-network-cluster for lua-lmod (>=8.7.55) -# opensuse-oss for lua53, lua53-luaterm,, sqlite3-tcl, tcl -# opensuse-devel-languages-lua for lua53-luafilesystem, lua53-luaposix -ARG JENKINS_URL -RUN if [ -n "$JENKINS_URL" ]; then \ - dnf -y remove lua-lmod; \ - dnf -y --nogpgcheck install lua-lmod --repo '*lua*' \ - --repo '*network-cluster*' --repo '*oss-proxy*' && \ - dnf clean all; \ - fi - # Add DAOS users ARG UID=1000 RUN --mount=type=bind,source=utils/scripts/helpers/daos-server-user-setup.sh,target=/tmp/daos/daos-server-user-setup.sh \ diff --git a/utils/scripts/helpers/repo-helper-leap15.sh b/utils/scripts/helpers/repo-helper-leap15.sh index 60e9ee9dda9..7feae5aacdc 100755 --- a/utils/scripts/helpers/repo-helper-leap15.sh +++ b/utils/scripts/helpers/repo-helper-leap15.sh @@ -76,7 +76,7 @@ install_dnf() { install_optional_ca() { ca_storage="/etc/pki/trust/anchors/" if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then - curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + curl -k -sSf -o "${ca_storage}lab_ca_file.crt" \ "$DAOS_LAB_CA_FILE_URL" update-ca-certificates fi @@ -99,7 +99,7 @@ if [ -n "$REPO_FILE_URL" ]; then install_optional_ca mkdir -p "$repos_dir" pushd "$repos_dir" - curl -k --noproxy '*' -sSf -o "daos_ci-leap${MAJOR_VER}-${REPOSITORY_NAME}.repo" \ + curl -sSf -o "daos_ci-leap${MAJOR_VER}-${REPOSITORY_NAME}.repo" \ "${REPO_FILE_URL}daos_ci-leap${MAJOR_VER}-${REPOSITORY_NAME}.repo" disable_repos "$repos_dir" popd @@ -107,15 +107,26 @@ if [ -n "$REPO_FILE_URL" ]; then # # when using a local repository. # unset HTTPS_PROXY # unset https_proxy - install_dnf else - if ! command -v dnf; then - zypper --non-interactive --gpg-auto-import-keys install \ - dnf dnf-plugins-core + # shellcheck disable=SC1091 + . /etc/os-release + + # network:cluster is only published for some Leap versions (e.g. not + # 15.4). Probe the GPG key URL first so unsupported versions (such as + # GitHub Actions using an older Leap) build without this repo instead + # of failing outright. + network_cluster_key_url="https://download.opensuse.org/repositories/network:/cluster/${VERSION_ID}/repodata/repomd.xml.key" + if curl -fsS --connect-timeout 5 --max-time 10 \ + "${network_cluster_key_url}" > /dev/null 2>&1; then + zypper --non-interactive addrepo --gpgcheck \ + "https://download.opensuse.org/repositories/network:/cluster/${VERSION_ID}/" \ + network-cluster + rpm --import "${network_cluster_key_url}" else - install_dnf + echo "Skipping network-cluster repo: ${network_cluster_key_url} is unavailable for ${VERSION_ID}" fi fi +install_dnf if [ ! -d /etc/yum.repos.d/ ]; then mkdir -p /etc/yum.repos.d/ pushd "$repos_dir" @@ -188,30 +199,51 @@ if [ -n "$REPO_FILE_URL" ]; then # Setup pip/uv to use the proxy only when the endpoint is reachable. pypi_proxy_url="${trusted_base_url}/api/pypi/pypi-proxy/simple" - if curl -k --noproxy '*' -fsS --connect-timeout 5 --max-time 10 \ + if curl -fsS --connect-timeout 5 --max-time 10 \ "${pypi_proxy_url}" > /dev/null 2>&1; then - cat < /etc/pip.conf -[global] - trusted-host = ${trusted_host} - index-url = ${pypi_proxy_url} - progress_bar = off - no_color = true - quiet = 1 -EOF + { + echo '[global]' + echo " trusted-host = ${trusted_host}" + echo " index-url = ${pypi_proxy_url}" + } > /etc/pip.conf + elif [ -n "${DAOS_HTTPS_PROXY:-}" ]; then + { + echo '[global]' + echo " proxy = ${DAOS_HTTPS_PROXY}" + } > /etc/pip.conf else - echo "Skipping pip proxy setup: ${pypi_proxy_url} is unreachable" + echo '[global]' > /etc/pip.conf fi + # Options common to every /etc/pip.conf variant above. + { + echo ' progress_bar = off' + echo ' no_color = true' + echo ' quiet = 1' + } >> /etc/pip.conf # Setup RubyGems to use artifactory/repository as the installation source only # when the endpoint is reachable. gem_proxy_url="${trusted_base_url}/api/gems/rubygems-proxy/" - if curl -k --noproxy '*' -fsS --connect-timeout 5 --max-time 10 \ + # Remove any stale rpmorig so a later ruby-devel install cannot leave it + # behind under a different name, breaking the restore of this gemrc. + rm -f /etc/gemrc.rpmorig + # Options common to every /etc/gemrc variant below. + { + echo '---' + echo 'gem: --no-document --quiet' + } > /etc/gemrc + if curl -fsS --connect-timeout 5 --max-time 10 \ "${gem_proxy_url}" > /dev/null 2>&1; then - cat < /etc/gemrc -:sources: -- ${gem_proxy_url} -EOF + { + echo ':sources:' + echo "- ${gem_proxy_url}" + } >> /etc/gemrc + elif [ -n "${DAOS_HTTPS_PROXY:-}" ]; then + { + echo "http_proxy: ${DAOS_HTTPS_PROXY}" + echo "https_proxy: ${DAOS_HTTPS_PROXY}" + } >> /etc/gemrc else - echo "Skipping /etc/gemrc setup: ${gem_proxy_url} is unreachable" + echo "Skipping gem source/proxy setup: ${gem_proxy_url} is unreachable" fi fi diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 68091f27f57..a6d253562bb 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -70,6 +70,25 @@ dnf --nodocs install ${dnf_install_args} \ which \ yasm +# According to https://pkgs.org/search/?q=lua-lmod, Leap 15.6 only has +# lua-lmod-8.7.34. +# This 8.7.34 version has a problem with loading modules, as described in +# https://github.com/TACC/Lmod/issues/687. +# This affects MPI detection in scons, so DAOS cannot be built with MPI support. +# A custom source of the package is required in CI to install a valid version +# of lua-lmod and its dependencies: +# opensuse-network-cluster for lua-lmod (>=8.7.55) +# opensuse-oss for lua53, lua53-luaterm,, sqlite3-tcl, tcl +# opensuse-devel-languages-lua for lua53-luafilesystem, lua53-luaposix +# repo-helper-leap15.sh ensures the network-cluster (and, when using +# an artifact server, oss-proxy) repos are present before this script runs. +if dnf repolist --disabled '*network-cluster*' 2>/dev/null | tail -n +2 | grep -q .; then + dnf -y remove lua-lmod + dnf -y --nogpgcheck install lua-lmod \ + --repo '*network-cluster*' \ + --repo '*oss-proxy*' +fi + if [[ "${INSTALL_BUILD_CI_ONLY:-}" != "true" ]]; then # Optional packages for full-featured images; can be skipped in essential-only mode. # shellcheck disable=SC2086 @@ -95,10 +114,16 @@ if [ "$arch" = x86_64 ]; then fi # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} ruby-devel +# ruby-devel ships its own default /etc/gemrc, which rpm installs in place +# of the one repo-helper-leap15.sh wrote, saving ours as /etc/gemrc.rpmorig. +if [ -f /etc/gemrc.rpmorig ]; then + mv /etc/gemrc.rpmorig /etc/gemrc +fi +# gem does not reliably auto-load /etc/gemrc on every distro/build; force it. +export GEMRC=/etc/gemrc gem install json -v 2.7.6 gem install dotenv -v 2.8.1 gem install fpm -v 1.16.0 if [ ! -f /usr/bin/fpm ]; then ln -s "$(basename "$(ls -1 /usr/bin/fpm.ruby*)")" /usr/bin/fpm fi -