From f78a9e0de7ca7d53c8d284170c936263a44cff42 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sun, 26 Jul 2026 15:21:11 +0530 Subject: [PATCH 1/4] fix(dist): replace lsof-based port check with ss/netstat//dev/tcp fallbacks and harden startup scripts Replace lsof with layered probe (ss, netstat, /dev/tcp) in server check_port to eliminate startup hang in Kubernetes pods where ulimit -n can be ~1 billion causing lsof to scan the entire FD table. Server util.sh: - Token-based listener matching with normalize_addr() for IPv6 canonicalization - IPv6 addresses canonicalized via getent ahosts (Linux) or python3 (macOS/BSD) - Authority extraction strips ? and # in addition to / and path - Same-family wildcard matching (IPv4/IPv6 isolated) - run_with_deadline() helper: deadline-bounded /dev/tcp without timeout - Atomic download(): temp file + mv on success, rm on failure - curl -fL with correct -o before -- argument ordering - wget progress flags in indexed array for set -u / Bash 3.2 safety - Helper functions scoped with _check_port_ prefix and unset -f cleanup - netstat -an fallback uses Bash built-in case-insensitive match - normalize_addr() wraps getent in timeout 2, tr fallback for case - process_num() returns boolean 0/1, process_id() echoes result - command_available() uses command -v instead of [[ -x ]] - get_ip() falls back to loopback on empty parse - free_memory() Darwin branch guards empty values - wait_for_startup() separates local/assignment, quotes http_code - read_property() localizes file_name/property_name - port_checked=1 set when ss/netstat succeeds and comparison is reliable PD and Store util.sh: - Remove dead check_port, fix command_available - Atomic download() matching server pattern - wget progress flags in indexed array - read_property() localizes file_name/property_name CI and test scripts: - test-check-port.sh runs as fast gate after Compile - fuser preflight on Linux, lsof removed - OS-aware port cleanup (fuser on Linux, safe lsof loop on macOS) - PD/Store/Server tests include fuser in non-Darwin prereqs - PD/Store tests call STOP_SCRIPT before manual cleanup - test-start-hugegraph.sh no longer masks java_home exit status Tests: test-check-port.sh - 40 mock-driven tests, 0 failures - timeout mock makes ss/netstat free cases deterministic - IPv6 canonicalization (compressed vs expanded) - download tests verify temp file + atomic rename + cleanup - wget success-path tests for PD and Store - Bash 3.2-safe array expansion throughout --- .github/workflows/server-ci.yml | 20 +- .../src/assembly/static/bin/util.sh | 149 +-- .../src/assembly/static/bin/util.sh | 475 ++++++++- .../src/assembly/travis/test-check-port.sh | 984 ++++++++++++++++++ .../travis/test-start-hugegraph-pd.sh | 29 +- .../travis/test-start-hugegraph-store.sh | 28 +- .../assembly/travis/test-start-hugegraph.sh | 34 +- .../static/bin/start-hugegraph-store.sh | 13 +- .../src/assembly/static/bin/util.sh | 176 ++-- 9 files changed, 1725 insertions(+), 183 deletions(-) create mode 100755 hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 586e36da49..402fbb8a2d 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -70,17 +70,30 @@ jobs: run: | mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + - name: Run check_port unit tests + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + # Validates ss/netstat//dev/tcp replacement for lsof + $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + - name: Check startup test prerequisites id: server-preflight if: ${{ env.BACKEND == 'rocksdb' }} run: | - for tool in lsof crontab curl java; do + # Note: lsof removed from prerequisites (replaced with ss/netstat//dev/tcp in check_port). + # This job always runs on Linux (ubuntu-22.04), which uses fuser for port cleanup. + for tool in crontab curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" exit 0 fi done + if ! command -v fuser >/dev/null 2>&1; then + echo "can_run=false" >> "$GITHUB_OUTPUT" + echo "skip_reason=missing tool: fuser (required for Linux port cleanup)" >> "$GITHUB_OUTPUT" + exit 0 + fi echo "can_run=true" >> "$GITHUB_OUTPUT" - name: Run start-hugegraph.sh foreground mode tests @@ -178,6 +191,11 @@ jobs: run: | mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + - name: Run check_port unit tests + # Validates ss/netstat//dev/tcp replacement for lsof + run: | + $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + - name: Run RocksDB core test run: | $TRAVIS_DIR/run-core-test.sh $BACKEND diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index 0b7ad0f0f0..e80255672e 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -20,20 +20,21 @@ # TODO: consider reuse it with server-dist module (almost same as it) function command_available() { local cmd=$1 - if [ $(command -v $cmd >/dev/null 2>&1) ]; then - return 1 - else + if command -v "$cmd" >/dev/null 2>&1; then return 0 fi + return 1 } # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." - property_name=$(echo $2 | sed 's/\./\\\./g') - cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 + property_name=$(echo "$2" | sed 's/\./\\\./g') + cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 } function write_property() { @@ -56,7 +57,7 @@ function parse_yaml() { local version=$2 local module=$3 - cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{ pre=""; for(i=1; i<=NF; ) { if(match($i, /version/)) { @@ -72,27 +73,19 @@ function parse_yaml() { } function process_num() { - num=`ps -ef | grep $1 | grep -v grep | wc -l` - return $num + local num + num=$(ps -ef | grep "$1" | grep -v grep | wc -l) + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { - pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` - return $pid -} - -# check the port of rest server is occupied -function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 - fi - lsof -i :$port >/dev/null - if [ $? -eq 0 ]; then - echo "The port $port has already been used" - exit 1 - fi + local pid + pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') + echo "$pid" + return 0 } function crontab_append() { @@ -130,13 +123,14 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=`date '+%s'` - local stop_s=$(( $now_s + $timeout_s )) + local now_s + now_s=$(date '+%s') + local stop_s=$(( now_s + timeout_s )) local status echo -n "Connecting to $server_name ($server_url)" - while [ $now_s -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -144,14 +138,14 @@ function wait_for_startup() { return 1 fi - status=`curl -o /dev/null -s -k -w %{http_code} $server_url` - if [[ $status -eq 200 || $status -eq 401 ]]; then + status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url") + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "The operation timed out when attempting to connect to $server_url" >&2 @@ -160,22 +154,28 @@ function wait_for_startup() { function free_memory() { local free="" - local os=`uname` + local os=$(uname) if [ "$os" == "Linux" ]; then - local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` - local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` - local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}') + local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}') + local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}') if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then echo "Failed to get free memory" exit 1 fi - free=`expr $mem_free + $mem_buffer + $mem_cached` - free=`expr $free / 1024` + free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") + free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_available=`expr $pages_free + $pages_inactive` - free=`expr $pages_available \* 4096 / 1024 / 1024` + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") + free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" exit 1 @@ -187,8 +187,8 @@ function calc_xmx() { local min_mem=$1 local max_mem=$2 # Get machine available memory - local free=`free_memory` - local half_free=$[free/2] + local free=$(free_memory) + local half_free=$((free/2)) local xmx=$min_mem if [[ "$free" -lt "$min_mem" ]]; then @@ -236,47 +236,74 @@ function ensure_path_writable() { } function get_ip() { - local os=`uname` + local os=$(uname) local loopback="127.0.0.1" local ip="" case $os in Linux) if command_available "ifconfig"; then - ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}') elif command_available "ip"; then - ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}') else ip=$loopback fi ;; FreeBSD|OpenBSD|Darwin) if command_available "ifconfig"; then - ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}') else ip=$loopback fi ;; SunOS) if command_available "ifconfig"; then - ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ') else ip=$loopback fi ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local link_url=$2 - if command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget ${link_url} -P ${path} $progress_opt - elif command_available "curl"; then - curl ${link_url} -o ${path}/${link_url} + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + local filename + filename=$(basename "${link_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + + if command_available "curl"; then + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi + elif command_available "wget"; then + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required wget or curl but they are unavailable" exit 1 @@ -289,19 +316,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d ${path}/${dir} ]; then - if [ ! -f ${path}/${tar} ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download ${path} ${link} - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi @@ -316,7 +341,7 @@ function wait_for_shutdown() { local pid="$2" local timeout_s="$3" - local now_s=`date '+%s'` + local now_s=$(date '+%s') local stop_s=$(( $now_s + $timeout_s )) echo -n "Killing $process_name(pid $pid)" >&2 @@ -328,7 +353,7 @@ function wait_for_shutdown() { return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 return 1 @@ -357,7 +382,7 @@ function kill_process() { return 0 fi - case "`uname`" in + case "$(uname)" in CYGWIN*) taskkill /F /PID "$pid" ;; *) kill "$pid" ;; esac diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index e2655c69d3..2714435927 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -18,11 +18,10 @@ function command_available() { local cmd=$1 - if [[ -x "$(command -v "$cmd")" ]]; then + if command -v "$cmd" >/dev/null 2>&1; then return 0 - else - return 1 fi + return 1 } # read a property from .properties file @@ -70,27 +69,417 @@ function parse_yaml() { } function process_num() { + local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - return "$num" + # Return 0 when no process, 1 when one or more. Using $num directly as an + # exit code would truncate values > 255, so treat this as a boolean result. + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { + local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - return "$pid" + echo "$pid" + return 0 +} + +# Run a command with a hard deadline via background watchdog. +# Returns the command's exit code if it finishes in time. +# If the deadline expires, the command is killed (exit code reflects signal). +# Works without the timeout command — uses sleep + kill -9 pattern. +function run_with_deadline() { + local cmd="$1" + local deadline="$2" + shift 2 + + bash -c "$cmd" bash "$@" & + local child_pid=$! + ( + local sleep_pid="" + cleanup_watchdog() { + [[ -n "$sleep_pid" ]] && kill -9 "$sleep_pid" 2>/dev/null + } + # Kill any direct children of the target PID (e.g. a spawned sleep) + # in case killing the wrapper left them orphaned. + kill_children() { + local child + for child in $(pgrep -P "$1" 2>/dev/null); do + kill -9 "$child" 2>/dev/null || true + done + } + trap 'cleanup_watchdog' EXIT TERM + sleep "$deadline" & sleep_pid=$! + wait "$sleep_pid" 2>/dev/null + if kill -0 "$child_pid" 2>/dev/null; then + kill -9 "$child_pid" 2>/dev/null + kill_children "$child_pid" + fi + ) 2>/dev/null & + local watchdog_pid=$! + + wait "$child_pid" 2>/dev/null + local rc=$? + # Kill the watchdog with SIGTERM so its EXIT trap reaps the sleep child. + kill -TERM "$watchdog_pid" 2>/dev/null || true + wait "$watchdog_pid" 2>/dev/null || true + return $rc +} + + +# Normalize an IP address to a canonical string for comparison. +# - IPv4 addresses are returned unchanged. +# - IPv6 addresses are stripped of brackets/zone scope, lowercased and +# compressed to the canonical textual form (via getent ahosts). +# - IPv4-mapped IPv6 (::ffff:1.2.3.4) is collapsed to the IPv4 form. +# - Hostnames are returned unchanged; callers should resolve them first. +function normalize_addr() { + local addr="$1" + + # Strip brackets + if [[ "$addr" =~ ^\[.*\]$ ]]; then + addr="${addr#\[}" + addr="${addr%\]}" + fi + + # Drop IPv6 zone scope (e.g. 127.0.0.53%lo, fe80::1%eth0) + addr="${addr%%\%*}" + + # IPv4-mapped IPv6 -> IPv4 so a bound IPv4-mapped socket is compared + # against an IPv4-configured address. + if [[ "$addr" =~ ^::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + echo "${BASH_REMATCH[1]}" + return + fi + + # Plain IPv4 + if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$addr" + return + fi + + # IPv6: ask glibc for the canonical compressed form + if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + local norm + if command_available "timeout"; then + norm=$(timeout 2 getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') + else + norm=$(getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') + fi + if [[ -n "$norm" ]]; then + echo "$norm" + return + fi + # macOS/BSD fallback: python3's socket module can canonicalise + # numeric IPv6 even when getent is unavailable. + if command_available "python3"; then + norm=$(python3 -c 'import socket, sys; print(socket.inet_ntop(socket.AF_INET6, socket.inet_pton(socket.AF_INET6, sys.argv[1])))' "$addr" 2>/dev/null) + if [[ -n "$norm" ]]; then + echo "$norm" + return + fi + fi + # No canonicaliser available: at least normalise hex case so ss/netstat + # lowercase output still matches an uppercase configured address. + echo "$addr" | tr '[:upper:]' '[:lower:]' + return + fi + + # Fall back to the cleaned original + echo "$addr" } -# check the port of rest server is occupied +# check whether the REST server port is occupied function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 + local url="$1" + local host + local port + + # Strip leading/trailing whitespace from URL (handles whitespace from ServerOptions) + url="${url#"${url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + + # Extract authority: strip scheme and stop at the first /, ? or #. + local authority + authority="${url#*://}" + authority="${authority%%[/?#]*}" + + # Extract host and port from authority. + if [[ "$authority" =~ ^\[([^\]]*)\]:([0-9]+)$ ]]; then + # IPv6 with port: [::1]:8080 + host="${BASH_REMATCH[1]}" + port="${BASH_REMATCH[2]}" + elif [[ "$authority" =~ ^\[([^\]]*)\]$ ]]; then + # IPv6 without port: [::1] + host="${BASH_REMATCH[1]}" + port="" + elif [[ "$authority" =~ :([0-9]+)$ ]]; then + # IPv4 or hostname with port: 127.0.0.1:8080, localhost:8080 + port="${BASH_REMATCH[1]}" + host="${authority%:*}" + else + # No explicit port in authority + host="$authority" + port="" fi - lsof -i :"$port" >/dev/null - if [ $? -eq 0 ]; then + + # Handle default ports from scheme when no explicit port is given + if [[ -z "$port" ]]; then + if [[ "$url" == https://* ]]; then + port="443" + elif [[ "$url" == http://* ]]; then + port="80" + else + return 0 + fi + fi + + # Validate port as a decimal number + if ! [[ "$port" =~ ^[0-9]+$ ]]; then + return 0 + fi + port=$((10#$port)) + if (( port < 1 || port > 65535 )); then + return 0 + fi + + # Strip any leading/trailing whitespace from host + host="${host#"${host%%[![:space:]]*}"}" + host="${host%"${host##*[![:space:]]}"}" + + local norm_host + norm_host=$(normalize_addr "$host") + + # Determine the address family of the configured host so we only treat + # same-family wildcard listeners as conflicts (IPv4 vs IPv6 sockets are + # separate unless explicitly dual-stacked). + local host_family="" + if [[ "$norm_host" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + host_family="ipv4" + elif [[ "$norm_host" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + host_family="ipv6" + else + host_family="hostname" + fi + + local in_use=0 + local port_checked=0 + + # Wildcard binds are detected by looking for any listener on the port. + # Specific hosts are matched against the local address; if the listener + # is a same-family wildcard (0.0.0.0 / :: / *), that is a conflict too. + local is_wildcard=0 + if [[ -z "$host" || "$host" == "0.0.0.0" || "$host" == "::" || "$host" == "*" ]]; then + is_wildcard=1 + fi + + # Build the list of acceptable normalized addresses for a specific host. + # Hostnames must be resolved to numeric IPs first because ss/netstat + # output is always numeric. + local candidate_addrs="" + if [[ $is_wildcard -eq 0 ]]; then + if [[ "$host_family" == "ipv4" || "$host_family" == "ipv6" ]]; then + # Already numeric + candidate_addrs="$norm_host" + else + # Hostname: resolve with deadline + if command_available "getent" && command_available "timeout"; then + candidate_addrs=$(timeout 2 getent ahosts "$host" 2>/dev/null | awk '{print $1}') + elif command_available "dscacheutil" && command_available "timeout"; then + candidate_addrs=$(timeout 2 dscacheutil -q host -a name "$host" 2>/dev/null \ + | awk '/ip_address:/{print $2}') + fi + fi + fi + + # Helper: scan a line of listener-table output and return 0 if it matches + # the configured host/port. Sets 'matched_token' to 1 when it evaluates a + # token that we can trust, so callers know whether "no match" is reliable. + local out line listener_addr norm_listener token matched_token + + # Returns true if the listener address is a wildcard on the same family + # as the configured host (or any family, if the host is a wildcard). + _check_port_wildcard_conflicts() { + local wl_addr="$1" + if [[ $is_wildcard -eq 1 ]]; then + return 0 + fi + # BSD netstat prints *. for an any-family wildcard. + if [[ "$wl_addr" == "*" ]]; then + return 0 + fi + if [[ "$wl_addr" == "0.0.0.0" ]]; then + [[ "$host_family" == "ipv4" ]] && return 0 + # A hostname that resolved to IPv4 can also bind an IPv4 wildcard + if [[ "$host_family" == "hostname" ]]; then + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + return 0 + fi + done <<< "$candidate_addrs" + fi + fi + if [[ "$wl_addr" == "::" ]]; then + [[ "$host_family" == "ipv6" ]] && return 0 + if [[ "$host_family" == "hostname" ]]; then + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + return 0 + fi + done <<< "$candidate_addrs" + fi + fi + return 1 + } + + _check_port_match_listener_line() { + local out_line="$1" + # Skip empty or whitespace-only lines before read -a to avoid an empty + # array expansion under set -u on Bash 3.2. + if [[ -z "${out_line//[[:space:]]/}" ]]; then + return 1 + fi + local -a tokens + read -r -a tokens <<< "$out_line" + for token in ${tokens[@]+"${tokens[@]}"}; do + # We only care about the "local address:port" token. For the + # common tools this is the first token that ends with the + # target port after ':' or '.' (peer addresses use :* on Linux + # and *.* on BSD, so they never match a numeric port). + if [[ "$token" =~ ^(.*):(${port})$ ]]; then + listener_addr="${BASH_REMATCH[1]}" + elif [[ "$token" =~ ^(.*)\.(${port})$ ]]; then + listener_addr="${BASH_REMATCH[1]}" + else + continue + fi + + # Wildcard host: any listener on this port is a conflict. + if [[ $is_wildcard -eq 1 ]]; then + matched_token=1 + return 0 + fi + + # No resolved candidates (hostname resolution failed/unsupported): + # we cannot reliably compare against the numeric listener table. + if [[ -z "$candidate_addrs" ]]; then + continue + fi + + matched_token=1 + norm_listener=$(normalize_addr "$listener_addr") + + # A same-family wildcard listener on this port conflicts with any + # specific host of that family. + if _check_port_wildcard_conflicts "$norm_listener"; then + return 0 + fi + + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$norm_listener" == "$(normalize_addr "$addr")" ]]; then + return 0 + fi + done <<< "$candidate_addrs" + done + return 1 + } + + if command_available "ss"; then + matched_token=0 + if out=$(ss -ltn 2>/dev/null); then + while IFS= read -r line; do + if _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + # ss -ltn succeeded. We can trust a non-match when we have a + # numeric host or resolved addresses (or a wildcard); otherwise + # fall through to /dev/tcp for unresolved hostnames. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + fi + fi + + if [[ $in_use -eq 0 && $port_checked -eq 0 ]] && command_available "netstat"; then + matched_token=0 + if out=$(netstat -ltn 2>/dev/null) && echo "$out" | grep -qi "listen"; then + while IFS= read -r line; do + if _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + # netstat -ltn output is the complete Linux listener table. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + elif out=$(netstat -an 2>/dev/null) && [[ -n "$out" ]]; then + local old_nocasematch + old_nocasematch=$(shopt -p nocasematch 2>/dev/null || true) + shopt -s nocasematch 2>/dev/null || true + while IFS= read -r line; do + if [[ "$line" == *listen* ]] && _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + eval "$old_nocasematch" 2>/dev/null || true + # netstat -an output is the complete BSD listener table. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + fi + fi + + if [[ $in_use -eq 0 && $port_checked -eq 0 ]]; then + # Probe the actual configured endpoint(s) with a short deadline. + local probe_addrs="$candidate_addrs" + if [[ -z "$probe_addrs" ]]; then + # Could not resolve (or wildcard with only loopback probe needed) + if [[ $is_wildcard -eq 1 ]]; then + probe_addrs="127.0.0.1 ::1" + else + probe_addrs="$host" + fi + fi + + local addr + for addr in $probe_addrs; do + # /dev/tcp needs unbracketed, normalized addresses + addr=$(normalize_addr "$addr") + [[ -z "$addr" ]] && continue + if command_available "timeout"; then + if timeout 1 bash -c ': >/dev/tcp/"$1"/"$2"' _ "$addr" "$port" 2>/dev/null; then + in_use=1 + break + fi + else + if run_with_deadline ': >/dev/tcp/"$1"/"$2" 2>/dev/null' 2 "$addr" "$port"; then + in_use=1 + break + fi + fi + done + fi + + local _rc=0 + if [[ "$in_use" -eq 1 ]]; then echo "The port $port has already been used" - exit 1 + _rc=1 fi + unset -f _check_port_wildcard_conflicts _check_port_match_listener_line || true + [[ $_rc -eq 1 ]] && exit 1 + return 0 } function crontab_append() { @@ -128,14 +517,15 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=$(date '+%s') + local now_s + now_s=$(date '+%s') local stop_s=$((now_s + timeout_s)) local status local error_file_name="startup_error.txt" echo -n "Connecting to $server_name ($server_url)" - while [ "$now_s" -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -147,7 +537,7 @@ function wait_for_startup() { fi status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") - if [[ $status -eq 200 || $status -eq 401 ]]; then + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" if [ -e "$error_file_name" ]; then @@ -180,9 +570,15 @@ function free_memory() { free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - local pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - local pages_available=$(expr "$pages_free" + "$pages_inactive") + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" @@ -273,23 +669,46 @@ function get_ip() { ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local download_url=$2 + + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + # Strip query/fragment so the on-disk name matches the server-side artifact. + local filename + filename=$(basename "${download_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + if command_available "curl"; then - if [ ! -d "$path" ]; then - mkdir -p "$path" || { - echo "Failed to create directory: $path" - exit 1 - } + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${download_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 fi - curl -L "${download_url}" -o "${path}/$(basename "${download_url}")" elif command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget "${download_url}" -P "${path}" $progress_opt + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${download_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required curl or wget but they are unavailable" exit 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh new file mode 100755 index 0000000000..c886f6f67e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -0,0 +1,984 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# test-check-port.sh — Unit tests for check_port() in util.sh +# +# Strategy: each test runs check_port in a subshell that overrides +# command_available() to control which probe branch is taken, and +# overrides the tool functions (ss, netstat, timeout) to control +# what they return — no real network connections needed. +# +# check_port calls `exit 1` when the port is in use, so the subshell +# exits 1; it returns normally (exit 0) when the port is free. +# +# Usage: ./test-check-port.sh [path-to-hugegraph-static-dir] +# path-to-hugegraph-static-dir: directory containing bin/util.sh +# Defaults to current directory. +# In CI: $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + +set -uo pipefail +# -u: fail on undefined variables (catches typos in test assertions) +# -o pipefail: pipeline exit status is the last non-zero component +# shellcheck disable=SC1090,SC1091 # UTIL_SH / PD_UTIL_SH sourced dynamically at runtime + +STATIC_DIR="${1:-$(pwd)}" +UTIL_SH="$STATIC_DIR/bin/util.sh" + +REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +PD_UTIL_SH="$REPO_ROOT/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh" +STORE_UTIL_SH="$REPO_ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh" + +PASS=0 +FAIL=0 +ERRORS=() + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +pass() { echo -e "${GREEN} PASS${NC} $1"; PASS=$((PASS + 1)); } +fail() { echo -e "${RED} FAIL${NC} $1"; ERRORS+=("$1"); FAIL=$((FAIL + 1)); } +section() { echo ""; echo "── $1 ──"; } + +# timeout mock used by ss/netstat test cases. It passes getent through so +# normalize_addr can canonicalise numeric IPv6, and pretends every other +# probe (the /dev/tcp fallback) failed, so free-port assertions are not +# influenced by real network state. +timeout() { + if [[ "${2:-}" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi +} + +echo "" +echo "check_port() unit test suite" +echo "util.sh: $UTIL_SH" +echo "" + +if [[ ! -f "$UTIL_SH" ]]; then + echo -e "${RED}ERROR:${NC} $UTIL_SH not found." + echo " Pass the HugeGraph static assembly dir as \$1" + exit 1 +fi + +# ── ss branch ───────────────────────────────────────────────────────────────── + +section "ss branch — IPv4" + +( + # shellcheck source=/dev/null + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: IPv4 port occupied → exit 1" \ + || fail "ss: IPv4 port occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:9090 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: IPv4 port free → exit 0" \ + || fail "ss: IPv4 port free → expected exit 0, got 1" + +section "ss branch — IPv6 URL with scheme (http://[::1]:8080)" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: http://[::1]:8080 occupied → exit 1" \ + || fail "ss: http://[::1]:8080 occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: http://[::1]:8080 free → exit 0" \ + || fail "ss: http://[::1]:8080 free → expected exit 0, got 1" + +section "ss branch — IPv6 URL without scheme ([::1]:8080)" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: [::1]:8080 (no scheme) occupied → exit 1" \ + || fail "ss: [::1]:8080 (no scheme) occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: [::1]:8080 (no scheme) free → exit 0" \ + || fail "ss: [::1]:8080 (no scheme) free → expected exit 0, got 1" + +section "ss branch — wildcard 0.0.0.0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } + check_port "http://0.0.0.0:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: 0.0.0.0:8080 occupied → exit 1" \ + || fail "ss: 0.0.0.0:8080 occupied → expected exit 1, got 0" + +section "ss branch — wildcard ::" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "http://[::]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: [::]:8080 occupied → exit 1" \ + || fail "ss: [::]:8080 occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "http://[::]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: [::]:8080 free → exit 0" \ + || fail "ss: [::]:8080 free → expected exit 0, got 1" + +# ── netstat branch ──────────────────────────────────────────────────────────── + +section "netstat branch — Linux format (-ltn), occupied" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "netstat -ltn: port 8080 occupied → exit 1" \ + || fail "netstat -ltn: port 8080 occupied → expected exit 1, got 0" + +section "netstat branch — Linux format (-ltn), free" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "netstat -ltn: port 8080 free → exit 0" \ + || fail "netstat -ltn: port 8080 free → expected exit 0, got 1" + +section "netstat branch — BSD/macOS fallback (-an), occupied" + +# Simulate netstat that produces no output for -ltn (Linux flag unsupported) +# but outputs BSD-format lines for -an +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { + if [[ "$1" == "-ltn" ]]; then + return 1 # flag not supported on BSD + fi + echo "tcp4 0 0 *.8080 *.* LISTEN" + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "netstat -an BSD: port 8080 occupied → exit 1" \ + || fail "netstat -an BSD: port 8080 occupied → expected exit 1, got 0" + +section "netstat branch — IP octet false-positive guard" + +# Port 80 check; netstat output contains 192.168.80.1:443 +# The .80 in the IP address must NOT match port 80 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 192.168.80.1:443 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:80" +) +[[ $? -eq 0 ]] \ + && pass "netstat: IP octet .80 does not false-positive for port 80 → exit 0" \ + || fail "netstat: IP octet .80 false-positived for port 80 → expected exit 0, got 1" + +section "ss branch — host dot-escaping guard" + +# Host 127.0.0.1 must be matched literally: a listener whose address merely +# matches the pattern with '.' as a regex wildcard (127a0b0c1) must NOT count +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 127a0b0c1:8080 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: unescaped-dot lookalike 127a0b0c1 does not false-positive → exit 0" \ + || fail "ss: unescaped-dot lookalike 127a0b0c1 false-positived → expected exit 0, got 1" + +# ── /dev/tcp fallback branch ────────────────────────────────────────────────── + +section "/dev/tcp fallback — timeout available, port occupied" + +# timeout exits 0 → connection succeeded → port in use +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + timeout() { + # Assert correct invocation: timeout 1 bash -c SCRIPT _ HOST PORT + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*"; return 2; } + return 0 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "/dev/tcp+timeout: connection succeeded (exit 0) → port occupied → exit 1" \ + || fail "/dev/tcp+timeout: connection succeeded → expected exit 1, got 0" + +section "/dev/tcp fallback — timeout available, port free" + +# timeout exits 1 → connection refused → port free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + timeout() { + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*"; return 2; } + return 1 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "/dev/tcp+timeout: connection refused (exit 1) → port free → exit 0" \ + || fail "/dev/tcp+timeout: connection refused → expected exit 0, got 1" + +section "/dev/tcp fallback — real loopback (ephemeral port)" + +# Start Python server on ephemeral port 0, capture actual port from child stdout +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + # Mock timeout: on hosts without timeout (macOS), run the probe directly. + # The probe args are: timeout 1 bash -c SCRIPT _ HOST PORT + timeout() { + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*" >&2; return 2; } + # Run the probe with a 2-second hard deadline via background + watchdog + bash -c "$4" "$5" "$6" "$7" 2>/dev/null & + local probe_pid=$! + (sleep 2; kill -9 "$probe_pid" 2>/dev/null) & + local watchdog_pid=$! + wait "$probe_pid" 2>/dev/null + local rc=$? + kill -9 "$watchdog_pid" 2>/dev/null + wait "$watchdog_pid" 2>/dev/null + return $rc + } + # Use a temp file to capture the bound port from child + port_file=$(mktemp) + trap 'rm -f "$port_file"; [[ -n "${PY_PID:-}" ]] && { kill -9 "$PY_PID" 2>/dev/null; wait "$PY_PID" 2>/dev/null; }' EXIT + + # Start Python server that prints the bound port to stdout + python3 -c " +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(('127.0.0.1', 0)) +s.listen(1) # actually listen so /dev/tcp can connect +port = s.getsockname()[1] +print(port, flush=True) +import time +time.sleep(30) # keep server alive +" > "$port_file" 2>/dev/null & + PY_PID=$! + # Poll for port file up to ~4s (Python cold-start can exceed 0.5s on busy CI) + bound_port="" + for _ in $(seq 1 40); do + bound_port=$(head -1 "$port_file" 2>/dev/null) + [[ -n "$bound_port" ]] && break + kill -0 $PY_PID 2>/dev/null || break + sleep 0.1 + done + if [[ -z "$bound_port" || ! "$bound_port" =~ ^[0-9]+$ ]]; then + echo "SKIP: failed to get bound port" + kill -9 $PY_PID 2>/dev/null || true + exit 77 + fi + # Verify child is alive + if ! kill -0 $PY_PID 2>/dev/null; then + echo "SKIP: Python child died" + exit 77 + fi + check_port "http://127.0.0.1:$bound_port" +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP /dev/tcp real ephemeral port (setup failed)" +elif [[ $_rc -eq 1 ]]; then + pass "/dev/tcp real ephemeral port → detects occupation → exit 1" +else + fail "/dev/tcp real ephemeral port → expected exit 1, got $_rc" +fi + +section "/dev/tcp fallback — no-timeout watchdog kills stuck probe" + +# run_with_deadline "sleep 5" 2 should return in ~2s, not ~5s. +# Proves the watchdog actually kills a stuck child, not just that the code path is taken. +( + source "$UTIL_SH" + start=$(date +%s) + run_with_deadline 'sleep 5' 2 + rc=$? + elapsed=$(($(date +%s) - start)) + # Must complete well before the 5s sleep (2s deadline + overhead) and return non-zero (killed) + if [[ $rc -ne 0 && $elapsed -le 4 ]]; then + exit 0 + else + echo "run_with_deadline: rc=$rc elapsed=${elapsed}s (expected rc≠0, ≤4s)" >&2 + exit 1 + fi +) +[[ $? -eq 0 ]] \ + && pass "/dev/tcp no-timeout watchdog: sleep 5 killed in ≤4s" \ + || fail "/dev/tcp no-timeout watchdog: watchdog did not kill in time" + +# ── host/port conflict semantics ────────────────────────────────────────────── + +section "Host/port conflict semantics" + +# Configured host 192.168.1.50, but only 127.0.0.1 is listening -> free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } + check_port "http://192.168.1.50:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IP configured vs loopback listener → free → exit 0" \ + || fail "specific IP configured vs loopback listener → expected free, got occupied" + +# Configured host 192.168.1.50, 0.0.0.0 is listening -> occupied +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } + check_port "http://192.168.1.50:8080" +) +[[ $? -eq 1 ]] \ + && pass "specific IP configured vs wildcard listener → occupied → exit 1" \ + || fail "specific IP configured vs wildcard listener → expected occupied, got free" + +# Specific IPv4 host should not conflict with IPv6 wildcard listener +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 [::]:8080 [::]:*"; return 0; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IPv4 host vs IPv6 wildcard listener → free → exit 0" \ + || fail "specific IPv4 host vs IPv6 wildcard listener → expected free, got occupied" + +# Specific IPv6 host should not conflict with IPv4 wildcard listener +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IPv6 host vs IPv4 wildcard listener → free → exit 0" \ + || fail "specific IPv6 host vs IPv4 wildcard listener → expected free, got occupied" + +# ── tool failure fallthrough ────────────────────────────────────────────────── + +section "Tool failure fallthrough" + +# ss fails, netstat succeeds and finds occupied port +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } + ss() { return 1; } # ss execution fails + netstat() { + if [[ "$1" == "-ltn" ]]; then echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; return 0; fi + return 1 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss exits 1 → fallthrough to netstat → occupied → exit 1" \ + || fail "ss exits 1 → fallthrough to netstat → expected occupied, got free" + +# both ss and netstat fail, /dev/tcp timeout path must still detect occupation +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } + ss() { return 1; } + netstat() { return 1; } + timeout() { return 0; } # /dev/tcp connect succeeds + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss/netstat both fail → /dev/tcp timeout path occupied → exit 1" \ + || fail "ss/netstat both fail → expected /dev/tcp fallback occupation, got free" + +# ── hostname and URL normalization ──────────────────────────────────────────── + +section "Hostname collision detection" + +# localhost should resolve to numeric addresses before ss matching +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } + timeout() { shift; "$@" 2>/dev/null; } # pass-through for mocked getent + getent() { + if [[ "$1" == "hosts" || "$1" == "ahosts" ]] && [[ "$2" == "localhost" ]]; then + echo "127.0.0.1" + return 0 + fi + return 1 + } + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } + check_port "http://localhost:8080" +) +[[ $? -eq 1 ]] \ + && pass "hostname localhost resolves to 127.0.0.1 and detects occupation → exit 1" \ + || fail "hostname localhost occupation was not detected via resolved address" + +# unresolved hostnames should skip ss/netstat text matching and use /dev/tcp fallback +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } + getent() { return 2; } # resolution fails + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } + timeout() { return 0; } # fallback connect succeeds + check_port "http://unresolved.localdomain:8080" +) +[[ $? -eq 1 ]] \ + && pass "unresolved hostname falls through to /dev/tcp fallback and detects occupation" \ + || fail "unresolved hostname should use /dev/tcp fallback" + +section "URL normalization and IPv6 hextet guard" + +# check_port should trim URL whitespace before parsing host/port +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } + check_port " http://127.0.0.1:8080 " +) +[[ $? -eq 1 ]] \ + && pass "leading/trailing URL whitespace is normalized → occupied detected" \ + || fail "URL whitespace normalization failed to detect occupied port" + +# Ensure ":8080" inside an IPv6 hextet does not match listener port 8080 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + timeout() { + if [[ "$2" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi + } + ss() { echo "tcp LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "IPv6 hextet containing 8080 does not false-positive port 8080" \ + || fail "IPv6 hextet false-positive: expected free, got occupied" + +# ── IPv6 address canonicalisation ───────────────────────────────────────────── + +section "IPv6 address canonicalisation" + +# getent mock that normalises the two spellings of loopback to ::1 +getent_canonical_loopback() { + if [[ "$1" == "ahosts" ]] && [[ "$2" == "::1" || "$2" == "0:0:0:0:0:0:0:1" ]]; then + echo "::1" + return 0 + fi + return 2 +} + +# Host ::1, listener expanded 0:0:0:0:0:0:0:1 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:8080 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "IPv6 host [::1] matches expanded listener [0:0:0:0:0:0:0:1]:8080" \ + || fail "IPv6 compressed vs expanded spelling did not match" + +# Host expanded 0:0:0:0:0:0:0:1, listener ::1 (mixed case) +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [::1]:8080 [::]:*"; } + check_port "http://[0:0:0:0:0:0:0:1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "IPv6 expanded host matches compressed listener" \ + || fail "IPv6 expanded vs compressed spelling did not match" + +# Host ::1, listener expanded on a different port → free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + timeout() { + if [[ "$2" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi + } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "IPv6 canonicalisation does not false-positive on different port" \ + || fail "IPv6 canonicalisation matched the wrong port" + +# ── edge cases ──────────────────────────────────────────────────────────────── + +section "Edge cases" + +# Empty URL → port string is empty → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "" +) +[[ $? -eq 0 ]] \ + && pass "empty URL → exit 0 (graceful)" \ + || fail "empty URL → expected exit 0, got 1" + +# Port out of range (> 65535) → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "http://127.0.0.1:99999" +) +[[ $? -eq 0 ]] \ + && pass "port 99999 (out of range) → exit 0 (graceful)" \ + || fail "port 99999 (out of range) → expected exit 0, got 1" + +# Port 0 (invalid) → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "http://127.0.0.1:0" +) +[[ $? -eq 0 ]] \ + && pass "port 0 (invalid) → exit 0 (graceful)" \ + || fail "port 0 (invalid) → expected exit 0, got 1" + +# Non-numeric port → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "127.0.0.1:abc" +) +[[ $? -eq 0 ]] \ + && pass "non-numeric port → exit 0 (graceful)" \ + || fail "non-numeric port → expected exit 0, got 1" + +# ── download() fallback ─────────────────────────────────────────────────────── + +section "download() atomic curl path (pd)" + +( + if [[ ! -f "$PD_UTIL_SH" ]]; then + exit 77 + fi + + source "$PD_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + CURL_CALLED=0 + curl() { + # Expected: curl -fL -o -- + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + CURL_CALLED=1 + # The temp name must be hidden and contain the PID. + if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "curl mock: unexpected output path: $3" + return 1 + fi + touch "$3" + return 0 + fi + echo "curl mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic curl path: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (PD) writes to temp and renames on success" +else + fail "download() (PD) atomic curl path failed" +fi + +section "download() atomic curl path (store)" + +( + if [[ ! -f "$STORE_UTIL_SH" ]]; then + exit 77 + fi + + source "$STORE_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + CURL_CALLED=0 + curl() { + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + CURL_CALLED=1 + if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "curl mock: unexpected output path: $3" + return 1 + fi + touch "$3" + return 0 + fi + echo "curl mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic curl path: store util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (Store) writes to temp and renames on success" +else + fail "download() (Store) atomic curl path failed" +fi + +section "download() atomic wget path (pd)" + +( + if [[ ! -f "$PD_UTIL_SH" ]]; then + exit 77 + fi + + source "$PD_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "curl" ]]; then return 1; fi + if [[ "$1" == "wget" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + WGET_CALLED=0 + wget() { + if [[ "$1" == "--help" ]]; then + # wget --help runs in a pipeline subshell, so side-effect counters + # cannot be observed from the parent. Only the stdout matters here. + echo "--show-progress" + return 0 + fi + # Expected: wget -q --show-progress -O -- + if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ + "$5" == "--" && \ + "$6" == "https://example.com/some/path/file.tar.gz" ]]; then + WGET_CALLED=1 + if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "wget mock: unexpected output path: $4" + return 1 + fi + touch "$4" + return 0 + fi + echo "wget mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic wget path: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (PD) wget writes to temp and renames on success" +else + fail "download() (PD) atomic wget path failed" +fi + +section "download() atomic wget path (store)" + +( + if [[ ! -f "$STORE_UTIL_SH" ]]; then + exit 77 + fi + + source "$STORE_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "curl" ]]; then return 1; fi + if [[ "$1" == "wget" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + WGET_CALLED=0 + wget() { + if [[ "$1" == "--help" ]]; then + # wget --help runs in a pipeline subshell, so side-effect counters + # cannot be observed from the parent. Only the stdout matters here. + echo "--show-progress" + return 0 + fi + # Expected: wget -q --show-progress -O -- + if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ + "$5" == "--" && \ + "$6" == "https://example.com/some/path/file.tar.gz" ]]; then + WGET_CALLED=1 + if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "wget mock: unexpected output path: $4" + return 1 + fi + touch "$4" + return 0 + fi + echo "wget mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic wget path: store util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (Store) wget writes to temp and renames on success" +else + fail "download() (Store) atomic wget path failed" +fi + +section "download() curl failure cleanup (pd)" + +# curl failure must clean up the partial file and never leave a poisoned artifact +( + if [[ ! -f "$PD_UTIL_SH" ]]; then exit 77; fi + source "$PD_UTIL_SH" + + OUTDIR=$(mktemp -d) + trap 'rm -rf "$OUTDIR"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + mkdir() { command mkdir -p "$2"; return 0; } + PARTIAL="" + curl() { + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + PARTIAL="$3" + # Simulate a partial/corrupted transfer: write something then fail. + echo "partial" > "$PARTIAL" + return 1 + fi + return 1 + } + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + rc=$? + + if [[ $rc -ne 0 && ! -f "$OUTDIR/file.tar.gz" && \ + ( -z "$PARTIAL" || ! -f "$PARTIAL" ) ]]; then + exit 0 + else + echo "rc=$rc PARTIAL=$PARTIAL DEST=$([[ -f $OUTDIR/file.tar.gz ]] && echo exists || echo missing)" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() curl failure cleanup: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() curl failure cleans temp and does not poison destination" +else + fail "download() curl failure did not clean partial file" +fi + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo "" +echo "════════════════════════════════" +echo -e " Results: ${GREEN}$PASS passed${NC} ${RED}$FAIL failed${NC}" +echo "════════════════════════════════" + +if [[ $FAIL -gt 0 ]]; then + echo "" + echo "Failed tests:" + for err in ${ERRORS[@]+"${ERRORS[@]}"}; do + echo -e " ${RED}✗${NC} $err" + done +fi + +echo "" +[[ $FAIL -eq 0 ]] && exit 0 || exit 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh index 260e7f5711..c6926c6dfd 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh @@ -30,6 +30,7 @@ set -uo pipefail PD_ROOT="${1:-$(pwd)}" BIN="$PD_ROOT/bin" START_SCRIPT="$BIN/start-hugegraph-pd.sh" +STOP_SCRIPT="$BIN/stop-hugegraph-pd.sh" PID_FILE="$BIN/pid" PD_URL="http://localhost:8620" STARTUP_WAIT=60 # seconds to wait for PD HTTP to respond @@ -67,14 +68,29 @@ section() { cleanup() { info "Cleaning up..." + "$STOP_SCRIPT" >/dev/null 2>&1 || true if [[ -s "$PID_FILE" ]]; then kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" rm -rf "$PD_ROOT/logs/" - # kill anything still holding the PD port - lsof -ti :8620 | xargs kill -9 2>/dev/null || true - lsof -ti :8686 | xargs kill -9 2>/dev/null || true + # kill anything still holding the PD port (fuser avoids lsof dependency) + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8620 8686; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + fuser -k 8620/tcp 2>/dev/null || true + fuser -k 8686/tcp 2>/dev/null || true + else + echo "Warning: Neither lsof nor fuser available for PD port cleanup" + fi sleep 3 } @@ -129,7 +145,12 @@ if [[ ! -f "$START_SCRIPT" ]]; then exit 1 fi -for tool in lsof curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof curl java" +else + _prereq_tools="fuser curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 7e11da28dd..9ff324ae29 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -51,15 +51,30 @@ section() { echo ""; echo "── $1 ──"; } cleanup() { info "Cleaning up..." + "$STOP_SCRIPT" >/dev/null 2>&1 || true if [[ -s "$PID_FILE" ]]; then kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" rm -rf "$STORE_ROOT/logs/" # kill anything holding Store ports (8520 REST, 8510 raft, 8500 gRPC) - lsof -ti :8520 | xargs kill -9 2>/dev/null || true - lsof -ti :8510 | xargs kill -9 2>/dev/null || true - lsof -ti :8500 | xargs kill -9 2>/dev/null || true + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8520 8510 8500; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + fuser -k 8520/tcp 2>/dev/null || true + fuser -k 8510/tcp 2>/dev/null || true + fuser -k 8500/tcp 2>/dev/null || true + else + echo "Warning: Neither lsof nor fuser available for Store port cleanup" + fi sleep 3 } @@ -110,7 +125,12 @@ if [[ ! -f "$START_SCRIPT" ]]; then exit 1 fi -for tool in lsof curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof curl java" +else + _prereq_tools="fuser curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index 353cab6518..c8e5bfced5 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -76,10 +76,26 @@ cleanup() { kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" - # kill anything still holding server ports so check_port doesn't fail next test - lsof -ti :8080 | xargs kill -9 2>/dev/null || true - lsof -ti :8182 | xargs kill -9 2>/dev/null || true - lsof -ti :8088 | xargs kill -9 2>/dev/null || true + # kill anything still holding server ports (fuser avoids lsof dependency) + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8080 8182 8088; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + # fuser exists and should work on Linux; suppress failures but log if debug needed + fuser -k 8080/tcp 2>/dev/null || true + fuser -k 8182/tcp 2>/dev/null || true + fuser -k 8088/tcp 2>/dev/null || true + else + # Neither lsof nor fuser available; ports may remain occupied + echo "Warning: Neither lsof nor fuser available for port cleanup" + fi sleep 3 crontab -l 2>/dev/null | grep -v monitor-hugegraph | crontab - 2>/dev/null || true } @@ -145,7 +161,12 @@ if [[ ! -f "$STOP_SCRIPT" ]]; then exit 1 fi -for tool in lsof crontab curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof crontab curl java" +else + _prereq_tools="fuser crontab curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 @@ -155,7 +176,8 @@ done # start-monitor.sh requires JAVA_HOME if [[ -z "${JAVA_HOME:-}" ]]; then if command -v /usr/libexec/java_home >/dev/null 2>&1; then - export JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)" + JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)" + export JAVA_HOME fi fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 5a11eeab39..88165e47b6 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -46,8 +46,8 @@ if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$TOP/bin/libjemalloc_aarch64.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" - if download_and_verify $download_url $lib_file $expected_md5; then - export LD_PRELOAD=$lib_file + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + export LD_PRELOAD="$lib_file" else echo "Failed to verify or download $lib_file, skip it" fi @@ -55,8 +55,8 @@ elif [[ $arch == "x86_64" ]]; then lib_file="$TOP/bin/libjemalloc.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" - if download_and_verify $download_url $lib_file $expected_md5; then - export LD_PRELOAD=$lib_file + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + export LD_PRELOAD="$lib_file" else echo "Failed to verify or download $lib_file, skip it" fi @@ -73,7 +73,8 @@ export FILE_LIMITN=1024 #export FILE_LIMITN=1024000 function check_evn_limit() { - local limit_check=$(ulimit -n) + local limit_check + limit_check=$(ulimit -n) if [[ ${limit_check} != "unlimited" && ${limit_check} -lt ${FILE_LIMITN} ]]; then echo -e "${BASH_SOURCE[0]##*/}:${LINENO}:\E[1;32m ulimit -n can open too few maximum file descriptors, need (${FILE_LIMITN})!! \E[0m" return 1 @@ -218,7 +219,7 @@ fi # JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml" #fi -if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then +if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then echo "HugeGraphStoreServer is already running..." exit 0 fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 3b3d660102..5e68d4cbb3 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -19,20 +19,21 @@ function command_available() { local cmd=$1 - if [ `command -v $cmd >/dev/null 2>&1` ]; then - return 1 - else + if command -v "$cmd" >/dev/null 2>&1; then return 0 fi + return 1 } # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." - property_name=`echo $2 | sed 's/\./\\\./g'` - cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 + property_name=$(echo "$2" | sed 's/\./\\\./g') + cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 } function write_property() { @@ -40,7 +41,7 @@ function write_property() { local key=$2 local value=$3 - local os=`uname` + local os=$(uname) case $os in # Note: in mac os should use sed -i '' "xxx" to replace string, # otherwise prompt 'command c expects \ followed by text'. @@ -55,7 +56,7 @@ function parse_yaml() { local version=$2 local module=$3 - cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{ pre=""; for(i=1; i<=NF; ) { if(match($i, /version/)) { @@ -71,27 +72,19 @@ function parse_yaml() { } function process_num() { - num=`ps -ef | grep $1 | grep -v grep | wc -l` - return $num + local num + num=$(ps -ef | grep "$1" | grep -v grep | wc -l) + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { - pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` - return $pid -} - -# check the port of rest server is occupied -function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 - fi - lsof -i :$port >/dev/null - if [ $? -eq 0 ]; then - echo "The port $port has already been used" - exit 1 - fi + local pid + pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') + echo "$pid" + return 0 } function crontab_append() { @@ -129,13 +122,14 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=`date '+%s'` - local stop_s=$(( $now_s + $timeout_s )) + local now_s + now_s=$(date '+%s') + local stop_s=$(( now_s + timeout_s )) local status echo -n "Connecting to $server_name ($server_url)" - while [ $now_s -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -143,14 +137,14 @@ function wait_for_startup() { return 1 fi - status=`curl -o /dev/null -s -k -w %{http_code} $server_url` - if [[ $status -eq 200 || $status -eq 401 ]]; then + status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url") + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "The operation timed out when attempting to connect to $server_url" >&2 @@ -159,22 +153,28 @@ function wait_for_startup() { function free_memory() { local free="" - local os=`uname` + local os=$(uname) if [ "$os" == "Linux" ]; then - local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` - local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` - local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}') + local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}') + local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}') if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then echo "Failed to get free memory" exit 1 fi - free=`expr $mem_free + $mem_buffer + $mem_cached` - free=`expr $free / 1024` + free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") + free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_available=`expr $pages_free + $pages_inactive` - free=`expr $pages_available \* 4096 / 1024 / 1024` + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") + free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" exit 1 @@ -186,8 +186,8 @@ function calc_xmx() { local min_mem=$1 local max_mem=$2 # Get machine available memory - local free=`free_memory` - local half_free=$[free/2] + local free=$(free_memory) + local half_free=$((free/2)) local xmx=$min_mem if [[ "$free" -lt "$min_mem" ]]; then @@ -235,47 +235,74 @@ function ensure_path_writable() { } function get_ip() { - local os=`uname` + local os=$(uname) local loopback="127.0.0.1" local ip="" case $os in Linux) if command_available "ifconfig"; then - ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}') elif command_available "ip"; then - ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}') else ip=$loopback fi ;; FreeBSD|OpenBSD|Darwin) if command_available "ifconfig"; then - ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}') else ip=$loopback fi ;; SunOS) if command_available "ifconfig"; then - ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ') else ip=$loopback fi ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local link_url=$2 - if command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget ${link_url} -P ${path} $progress_opt - elif command_available "curl"; then - curl ${link_url} -o ${path}/${link_url} + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + local filename + filename=$(basename "${link_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + + if command_available "curl"; then + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi + elif command_available "wget"; then + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required wget or curl but they are unavailable" exit 1 @@ -286,14 +313,15 @@ download_and_verify() { local url=$1 local filepath=$2 local expected_md5=$3 + local actual_md5 - if [[ -f $filepath ]]; then + if [[ -f "$filepath" ]]; then echo "File $filepath exists. Verifying MD5 checksum..." - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then + actual_md5=$(md5sum -- "$filepath" | awk '{ print $1 }') + if [[ "$actual_md5" != "$expected_md5" ]]; then echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" echo "Deleting $filepath..." - rm -f $filepath + rm -f -- "$filepath" else echo "MD5 checksum verification succeeded for $filepath." return 0 @@ -301,11 +329,17 @@ download_and_verify() { fi echo "Downloading $filepath..." - curl -L -o $filepath $url - - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + local tmp="${filepath}.tmp.$$" + if curl -fL -o "$tmp" -- "$url"; then + actual_md5=$(md5sum -- "$tmp" | awk '{ print $1 }') + if [[ "$actual_md5" != "$expected_md5" ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + rm -f -- "$tmp" + return 1 + fi + mv -f -- "$tmp" "$filepath" + else + rm -f -- "$tmp" return 1 fi @@ -318,19 +352,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d ${path}/${dir} ]; then - if [ ! -f ${path}/${tar} ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download ${path} ${link} - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi @@ -345,7 +377,7 @@ function wait_for_shutdown() { local pid="$2" local timeout_s="$3" - local now_s=`date '+%s'` + local now_s=$(date '+%s') local stop_s=$(( $now_s + $timeout_s )) echo -n "Killing $process_name(pid $pid)" >&2 @@ -357,7 +389,7 @@ function wait_for_shutdown() { return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 return 1 @@ -386,7 +418,7 @@ function kill_process() { return 0 fi - case "`uname`" in + case "$(uname)" in CYGWIN*) taskkill /F /PID "$pid" ;; *) kill "$pid" ;; esac From 986c95f19ddd22d29ecf9be257c4a2150cd2e1d9 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sun, 26 Jul 2026 15:45:22 +0530 Subject: [PATCH 2/4] fix(bin): polish server util helpers after lsof port-check review - Localize read_property variables (match PD/Store) - Use if ! download / if ! tar in ensure_package_exist so atomic download failures are detected without a fragile $? check Also merges upstream master so the branch is no longer diverged (RISC-V libatomic + CI from #3102 kept intact with check_port). --- .../hugegraph-dist/src/assembly/static/bin/util.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 5040daa652..687e5d2001 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -66,6 +66,8 @@ function configure_riscv64_libatomic() { # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." property_name=$(echo "$2" | sed 's/\./\\\./g') @@ -760,19 +762,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d "${path}"/"${dir}" ]; then - if [ ! -f "${path}"/"${tar}" ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download "${path}" "${link}" - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar zxvf "${path}"/"${tar}" -C "${path}" >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi From 5f4a70d1d4822a92b78e12b419bb954162d3a390 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 27 Jul 2026 20:13:26 +0530 Subject: [PATCH 3/4] fix(bin): converge check_port on a conservative port-only preflight Rework the startup port check to the minimal design agreed in review. The preflight is best effort - the server's own bind stays authoritative - so it now answers one question only: is anything already listening on this port? check_port: - Parse just enough of the configured URL to obtain a validated port: case-insensitive scheme, explicit or default port, scheme-less value, bracketed IPv6. Ambiguous unbracketed IPv6 is rejected with a warning instead of guessed. - Detect listeners per OS, inspecting only LISTEN rows and only the local-address column: ss -H -ltn on Linux, netstat -an -p tcp on BSD. Splitting on the last separator keeps IPv6 hextets from being read as the port. - Return busy, free or unknown. A probe that is missing, fails, or yields no recognisable listener row is unknown: warn and let the bind decide. - Drop DNS resolution, address canonicalisation, cross-family endpoint matching, /dev/tcp and its watchdog. Port-level detection covers the dual-stack and wildcard cases conservatively without reproducing kernel socket semantics in shell, and removes the last unbounded call from the startup path. This fixes the macOS false positive where netstat -ltn returned non-listening rows and an unrelated ESTABLISHED peer port aborted startup. Also in this pass: - store: a failed atomic rename no longer reports success, so LD_PRELOAD is never exported for a library that did not install. Same fix applied to download() in all three copies. - download(): mktemp instead of a PID-derived name, which is shared by concurrent background subshells. - wait_for_startup(): bound each probe with --connect-timeout / --max-time by the time left in the deadline, so one blackholed request cannot outlive the configured timeout. - pd-store-ci: preflight checked lsof while cleanup used fuser; align it. - Revert unrelated process_num / process_id / free_memory / get_ip changes. netstat is used as the Linux fallback rather than fuser: fuser cannot distinguish "not found" from "insufficient privileges", which would break the busy/free/unknown contract. Tests: test-check-port.sh replaced with a compact contract suite - URL to port, ss busy/free/failure, BSD LISTEN vs ESTABLISHED vs TIME_WAIT, unknown fallback, a real ephemeral listener, rename failure, and a bounded startup probe. 41 passed, 0 failed. Verified non-vacuous by mutation: dropping the LISTEN filter, downgrading unknown to free, and reading the foreign-address column each fail the suite. --- .github/workflows/pd-store-ci.yml | 8 +- .../src/assembly/static/bin/util.sh | 12 +- .../src/assembly/static/bin/util.sh | 525 ++------ .../src/assembly/travis/test-check-port.sh | 1120 ++++------------- .../src/assembly/static/bin/util.sh | 20 +- 5 files changed, 387 insertions(+), 1298 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 62006794c7..6f670e1cb9 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -110,7 +110,9 @@ jobs: - name: Check startup test prerequisites (PD) id: pd-preflight run: | - for tool in lsof curl java; do + # These jobs run on Linux, where the suites clean up ports with fuser. + # lsof is no longer required: check_port now uses ss/netstat. + for tool in fuser curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" @@ -190,7 +192,9 @@ jobs: - name: Check startup test prerequisites (Store) id: store-preflight run: | - for tool in lsof curl java; do + # These jobs run on Linux, where the suites clean up ports with fuser. + # lsof is no longer required: check_port now uses ss/netstat. + for tool in fuser curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index e80255672e..4f4a90b45f 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -280,15 +280,19 @@ function download() { } fi - local filename + local filename tmp filename=$(basename "${link_url%%[?#]*}") - local tmp="${path}/.${filename}.tmp.$$" local dest="${path}/${filename}" + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || { + echo "Failed to create a temporary file in $path" + return 1 + } if command_available "curl"; then # -o must appear before -- so it is parsed as an option, not an extra URL. if curl -fL -o "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -299,7 +303,7 @@ function download() { progress_opt=(-q --show-progress) fi if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 687e5d2001..c794bd6d2e 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -110,416 +110,150 @@ function parse_yaml() { } function process_num() { - local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - # Return 0 when no process, 1 when one or more. Using $num directly as an - # exit code would truncate values > 255, so treat this as a boolean result. - if (( num > 0 )); then - return 1 - fi - return 0 + return "$num" } function process_id() { - local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - echo "$pid" - return 0 -} - -# Run a command with a hard deadline via background watchdog. -# Returns the command's exit code if it finishes in time. -# If the deadline expires, the command is killed (exit code reflects signal). -# Works without the timeout command — uses sleep + kill -9 pattern. -function run_with_deadline() { - local cmd="$1" - local deadline="$2" - shift 2 - - bash -c "$cmd" bash "$@" & - local child_pid=$! - ( - local sleep_pid="" - cleanup_watchdog() { - [[ -n "$sleep_pid" ]] && kill -9 "$sleep_pid" 2>/dev/null - } - # Kill any direct children of the target PID (e.g. a spawned sleep) - # in case killing the wrapper left them orphaned. - kill_children() { - local child - for child in $(pgrep -P "$1" 2>/dev/null); do - kill -9 "$child" 2>/dev/null || true - done - } - trap 'cleanup_watchdog' EXIT TERM - sleep "$deadline" & sleep_pid=$! - wait "$sleep_pid" 2>/dev/null - if kill -0 "$child_pid" 2>/dev/null; then - kill -9 "$child_pid" 2>/dev/null - kill_children "$child_pid" - fi - ) 2>/dev/null & - local watchdog_pid=$! - - wait "$child_pid" 2>/dev/null - local rc=$? - # Kill the watchdog with SIGTERM so its EXIT trap reaps the sleep child. - kill -TERM "$watchdog_pid" 2>/dev/null || true - wait "$watchdog_pid" 2>/dev/null || true - return $rc -} - - -# Normalize an IP address to a canonical string for comparison. -# - IPv4 addresses are returned unchanged. -# - IPv6 addresses are stripped of brackets/zone scope, lowercased and -# compressed to the canonical textual form (via getent ahosts). -# - IPv4-mapped IPv6 (::ffff:1.2.3.4) is collapsed to the IPv4 form. -# - Hostnames are returned unchanged; callers should resolve them first. -function normalize_addr() { - local addr="$1" - - # Strip brackets - if [[ "$addr" =~ ^\[.*\]$ ]]; then - addr="${addr#\[}" - addr="${addr%\]}" - fi - - # Drop IPv6 zone scope (e.g. 127.0.0.53%lo, fe80::1%eth0) - addr="${addr%%\%*}" - - # IPv4-mapped IPv6 -> IPv4 so a bound IPv4-mapped socket is compared - # against an IPv4-configured address. - if [[ "$addr" =~ ^::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then - echo "${BASH_REMATCH[1]}" - return - fi - - # Plain IPv4 - if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "$addr" - return - fi - - # IPv6: ask glibc for the canonical compressed form - if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then - local norm - if command_available "timeout"; then - norm=$(timeout 2 getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') - else - norm=$(getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') - fi - if [[ -n "$norm" ]]; then - echo "$norm" - return - fi - # macOS/BSD fallback: python3's socket module can canonicalise - # numeric IPv6 even when getent is unavailable. - if command_available "python3"; then - norm=$(python3 -c 'import socket, sys; print(socket.inet_ntop(socket.AF_INET6, socket.inet_pton(socket.AF_INET6, sys.argv[1])))' "$addr" 2>/dev/null) - if [[ -n "$norm" ]]; then - echo "$norm" - return - fi - fi - # No canonicaliser available: at least normalise hex case so ss/netstat - # lowercase output still matches an uppercase configured address. - echo "$addr" | tr '[:upper:]' '[:lower:]' - return - fi - - # Fall back to the cleaned original - echo "$addr" + return "$pid" } -# check whether the REST server port is occupied -function check_port() { +# Extract a validated TCP port from a configured server URL. +# Echoes the port on success. Returns 1 when the value carries no usable port +# or is ambiguous, in which case the caller skips the preflight. +function parse_port_from_url() { local url="$1" - local host - local port - # Strip leading/trailing whitespace from URL (handles whitespace from ServerOptions) + # ServerOptions tolerates surrounding whitespace, so strip it first. url="${url#"${url%%[![:space:]]*}"}" url="${url%"${url##*[![:space:]]}"}" + [[ -z "$url" ]] && return 1 + + # The scheme is optional and case-insensitive. + local scheme="" rest="$url" + if [[ "$url" == *"://"* ]]; then + scheme=$(echo "${url%%://*}" | tr '[:upper:]' '[:lower:]') + rest="${url#*://}" + fi - # Extract authority: strip scheme and stop at the first /, ? or #. - local authority - authority="${url#*://}" - authority="${authority%%[/?#]*}" + # The authority ends at the first '/', '?' or '#'. + local authority="${rest%%[/?#]*}" + [[ -z "$authority" ]] && return 1 - # Extract host and port from authority. - if [[ "$authority" =~ ^\[([^\]]*)\]:([0-9]+)$ ]]; then - # IPv6 with port: [::1]:8080 - host="${BASH_REMATCH[1]}" + local port="" + if [[ "$authority" =~ ^\[[^]]*\](:([0-9]+))?$ ]]; then + # Bracketed IPv6, with or without a port: [::1] or [::1]:8080 port="${BASH_REMATCH[2]}" - elif [[ "$authority" =~ ^\[([^\]]*)\]$ ]]; then - # IPv6 without port: [::1] - host="${BASH_REMATCH[1]}" - port="" - elif [[ "$authority" =~ :([0-9]+)$ ]]; then - # IPv4 or hostname with port: 127.0.0.1:8080, localhost:8080 - port="${BASH_REMATCH[1]}" - host="${authority%:*}" - else - # No explicit port in authority - host="$authority" - port="" + elif [[ "$authority" == *:*:* ]]; then + # Unbracketed IPv6 is ambiguous: in "::1:8080" the trailing group may + # be a port or another hextet. Refuse to guess. + echo "WARN: ambiguous IPv6 authority '$authority' in server URL;" \ + "use bracket notation such as [::1]:8080." >&2 + return 1 + elif [[ "$authority" == *:* ]]; then + port="${authority##*:}" fi - # Handle default ports from scheme when no explicit port is given + # Fall back to the scheme's default port. if [[ -z "$port" ]]; then - if [[ "$url" == https://* ]]; then - port="443" - elif [[ "$url" == http://* ]]; then - port="80" - else - return 0 - fi + case "$scheme" in + http) port="80" ;; + https) port="443" ;; + *) return 1 ;; + esac fi - # Validate port as a decimal number - if ! [[ "$port" =~ ^[0-9]+$ ]]; then - return 0 - fi + [[ "$port" =~ ^[0-9]+$ ]] || return 1 + # Normalise leading-zero forms; Java reads 08080 as decimal 8080. port=$((10#$port)) - if (( port < 1 || port > 65535 )); then - return 0 - fi + (( port >= 1 && port <= 65535 )) || return 1 - # Strip any leading/trailing whitespace from host - host="${host#"${host%%[![:space:]]*}"}" - host="${host%"${host##*[![:space:]]}"}" - - local norm_host - norm_host=$(normalize_addr "$host") - - # Determine the address family of the configured host so we only treat - # same-family wildcard listeners as conflicts (IPv4 vs IPv6 sockets are - # separate unless explicitly dual-stacked). - local host_family="" - if [[ "$norm_host" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - host_family="ipv4" - elif [[ "$norm_host" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then - host_family="ipv6" - else - host_family="hostname" - fi - - local in_use=0 - local port_checked=0 - - # Wildcard binds are detected by looking for any listener on the port. - # Specific hosts are matched against the local address; if the listener - # is a same-family wildcard (0.0.0.0 / :: / *), that is a conflict too. - local is_wildcard=0 - if [[ -z "$host" || "$host" == "0.0.0.0" || "$host" == "::" || "$host" == "*" ]]; then - is_wildcard=1 - fi - - # Build the list of acceptable normalized addresses for a specific host. - # Hostnames must be resolved to numeric IPs first because ss/netstat - # output is always numeric. - local candidate_addrs="" - if [[ $is_wildcard -eq 0 ]]; then - if [[ "$host_family" == "ipv4" || "$host_family" == "ipv6" ]]; then - # Already numeric - candidate_addrs="$norm_host" - else - # Hostname: resolve with deadline - if command_available "getent" && command_available "timeout"; then - candidate_addrs=$(timeout 2 getent ahosts "$host" 2>/dev/null | awk '{print $1}') - elif command_available "dscacheutil" && command_available "timeout"; then - candidate_addrs=$(timeout 2 dscacheutil -q host -a name "$host" 2>/dev/null \ - | awk '/ip_address:/{print $2}') - fi - fi - fi - - # Helper: scan a line of listener-table output and return 0 if it matches - # the configured host/port. Sets 'matched_token' to 1 when it evaluates a - # token that we can trust, so callers know whether "no match" is reliable. - local out line listener_addr norm_listener token matched_token + echo "$port" +} - # Returns true if the listener address is a wildcard on the same family - # as the configured host (or any family, if the host is a wildcard). - _check_port_wildcard_conflicts() { - local wl_addr="$1" - if [[ $is_wildcard -eq 1 ]]; then +# Echo "busy", "free" or "unknown" for the given TCP port. +# +# Detection is deliberately port-only, matching the conservative behaviour of +# the `lsof -i :PORT` call this replaces. Reproducing kernel socket semantics +# in Bash - dual-stack IPV6_V6ONLY, wildcard versus specific binds, address +# canonicalisation - produced more wrong answers than it prevented, so we only +# ask "is anything already listening on this port?". +# +# Only LISTEN rows and only the local-address column are inspected, so an +# unrelated outbound connection to the same port number is never mistaken for +# a local listener. A tool that is missing, fails, or yields no recognisable +# listener row reports "unknown" rather than "free". +function port_listen_state() { + local port="$1" + local out + local os + os=$(uname) + + # $4 is the local address for both `ss -ltn` and BSD `netstat -an`. + # Splitting on the last separator keeps IPv6 hextets (for example + # [2001:db8::80]:443) from being misread as the port. + local parser=' + NF >= 4 && (!want_listen || $NF == "LISTEN") { + addr = $4 + cut = 0 + for (k = length(addr); k > 0; k--) { + if (substr(addr, k, 1) == sep) { cut = k; break } + } + if (cut == 0) next + rows++ + if (substr(addr, cut + 1) == port) { found = 1; exit } + } + END { + if (found) print "busy" + else if (rows > 0) print "free" + else print "unknown" + }' + + if [[ "$os" == "Darwin" || "$os" == *BSD* ]]; then + if command_available "netstat" && out=$(netstat -an -p tcp 2>/dev/null) \ + && [[ -n "$out" ]]; then + echo "$out" | awk -v port="$port" -v sep="." -v want_listen=1 "$parser" return 0 fi - # BSD netstat prints *. for an any-family wildcard. - if [[ "$wl_addr" == "*" ]]; then + else + # `ss -H -ltn` already restricts output to listening sockets. + if command_available "ss" && out=$(ss -H -ltn 2>/dev/null) && [[ -n "$out" ]]; then + echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=0 "$parser" return 0 fi - if [[ "$wl_addr" == "0.0.0.0" ]]; then - [[ "$host_family" == "ipv4" ]] && return 0 - # A hostname that resolved to IPv4 can also bind an IPv4 wildcard - if [[ "$host_family" == "hostname" ]]; then - local addr - while IFS= read -r addr; do - [[ -z "$addr" ]] && continue - if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - return 0 - fi - done <<< "$candidate_addrs" - fi - fi - if [[ "$wl_addr" == "::" ]]; then - [[ "$host_family" == "ipv6" ]] && return 0 - if [[ "$host_family" == "hostname" ]]; then - local addr - while IFS= read -r addr; do - [[ -z "$addr" ]] && continue - if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then - return 0 - fi - done <<< "$candidate_addrs" - fi - fi - return 1 - } - - _check_port_match_listener_line() { - local out_line="$1" - # Skip empty or whitespace-only lines before read -a to avoid an empty - # array expansion under set -u on Bash 3.2. - if [[ -z "${out_line//[[:space:]]/}" ]]; then - return 1 - fi - local -a tokens - read -r -a tokens <<< "$out_line" - for token in ${tokens[@]+"${tokens[@]}"}; do - # We only care about the "local address:port" token. For the - # common tools this is the first token that ends with the - # target port after ':' or '.' (peer addresses use :* on Linux - # and *.* on BSD, so they never match a numeric port). - if [[ "$token" =~ ^(.*):(${port})$ ]]; then - listener_addr="${BASH_REMATCH[1]}" - elif [[ "$token" =~ ^(.*)\.(${port})$ ]]; then - listener_addr="${BASH_REMATCH[1]}" - else - continue - fi - - # Wildcard host: any listener on this port is a conflict. - if [[ $is_wildcard -eq 1 ]]; then - matched_token=1 - return 0 - fi - - # No resolved candidates (hostname resolution failed/unsupported): - # we cannot reliably compare against the numeric listener table. - if [[ -z "$candidate_addrs" ]]; then - continue - fi - - matched_token=1 - norm_listener=$(normalize_addr "$listener_addr") - - # A same-family wildcard listener on this port conflicts with any - # specific host of that family. - if _check_port_wildcard_conflicts "$norm_listener"; then - return 0 - fi - - local addr - while IFS= read -r addr; do - [[ -z "$addr" ]] && continue - if [[ "$norm_listener" == "$(normalize_addr "$addr")" ]]; then - return 0 - fi - done <<< "$candidate_addrs" - done - return 1 - } - - if command_available "ss"; then - matched_token=0 - if out=$(ss -ltn 2>/dev/null); then - while IFS= read -r line; do - if _check_port_match_listener_line "$line"; then - in_use=1 - break - fi - done <<< "$out" - # ss -ltn succeeded. We can trust a non-match when we have a - # numeric host or resolved addresses (or a wildcard); otherwise - # fall through to /dev/tcp for unresolved hostnames. - if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then - port_checked=1 - fi + if command_available "netstat" && out=$(netstat -ltn 2>/dev/null) \ + && [[ -n "$out" ]]; then + echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=1 "$parser" + return 0 fi fi - if [[ $in_use -eq 0 && $port_checked -eq 0 ]] && command_available "netstat"; then - matched_token=0 - if out=$(netstat -ltn 2>/dev/null) && echo "$out" | grep -qi "listen"; then - while IFS= read -r line; do - if _check_port_match_listener_line "$line"; then - in_use=1 - break - fi - done <<< "$out" - # netstat -ltn output is the complete Linux listener table. - if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then - port_checked=1 - fi - elif out=$(netstat -an 2>/dev/null) && [[ -n "$out" ]]; then - local old_nocasematch - old_nocasematch=$(shopt -p nocasematch 2>/dev/null || true) - shopt -s nocasematch 2>/dev/null || true - while IFS= read -r line; do - if [[ "$line" == *listen* ]] && _check_port_match_listener_line "$line"; then - in_use=1 - break - fi - done <<< "$out" - eval "$old_nocasematch" 2>/dev/null || true - # netstat -an output is the complete BSD listener table. - if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then - port_checked=1 - fi - fi - fi + echo "unknown" +} - if [[ $in_use -eq 0 && $port_checked -eq 0 ]]; then - # Probe the actual configured endpoint(s) with a short deadline. - local probe_addrs="$candidate_addrs" - if [[ -z "$probe_addrs" ]]; then - # Could not resolve (or wildcard with only loopback probe needed) - if [[ $is_wildcard -eq 1 ]]; then - probe_addrs="127.0.0.1 ::1" - else - probe_addrs="$host" - fi - fi +# Best-effort startup preflight. Exits 1 when the configured port is already +# in use. The server's own bind stays authoritative, so an inconclusive +# result only warns and lets startup proceed. +function check_port() { + local url="$1" + local port + local state - local addr - for addr in $probe_addrs; do - # /dev/tcp needs unbracketed, normalized addresses - addr=$(normalize_addr "$addr") - [[ -z "$addr" ]] && continue - if command_available "timeout"; then - if timeout 1 bash -c ': >/dev/tcp/"$1"/"$2"' _ "$addr" "$port" 2>/dev/null; then - in_use=1 - break - fi - else - if run_with_deadline ': >/dev/tcp/"$1"/"$2" 2>/dev/null' 2 "$addr" "$port"; then - in_use=1 - break - fi - fi - done - fi + port=$(parse_port_from_url "$url") || return 0 + + state=$(port_listen_state "$port") + case "$state" in + busy) + echo "The port $port has already been used" + exit 1 + ;; + unknown) + echo "WARN: could not determine whether port $port is free;" \ + "continuing and letting the server bind decide." >&2 + ;; + esac - local _rc=0 - if [[ "$in_use" -eq 1 ]]; then - echo "The port $port has already been used" - _rc=1 - fi - unset -f _check_port_wildcard_conflicts _check_port_match_listener_line || true - [[ $_rc -eq 1 ]] && exit 1 return 0 } @@ -577,7 +311,13 @@ function wait_for_startup() { return 1 fi - status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") + # Bound each probe by the time left in the overall deadline: without + # --max-time a single blackholed request blocks past ${timeout_s}s. + local remain_s=$((stop_s - now_s)) + [ "$remain_s" -lt 1 ] && remain_s=1 + local connect_s=$((remain_s < 5 ? remain_s : 5)) + status=$(curl -I -sS -k --connect-timeout "$connect_s" --max-time "$remain_s" \ + -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" @@ -611,15 +351,9 @@ function free_memory() { free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free pages_inactive - pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then - echo "Failed to get free memory" - exit 1 - fi - local pages_available - pages_available=$(expr "$pages_free" + "$pages_inactive") + local pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + local pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + local pages_available=$(expr "$pages_free" + "$pages_inactive") free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" @@ -710,8 +444,7 @@ function get_ip() { ;; *) ip=$loopback;; esac - [[ -z "$ip" ]] && ip=$loopback - echo "$ip" + echo $ip } function download() { @@ -726,15 +459,19 @@ function download() { fi # Strip query/fragment so the on-disk name matches the server-side artifact. - local filename + local filename tmp filename=$(basename "${download_url%%[?#]*}") - local tmp="${path}/.${filename}.tmp.$$" local dest="${path}/${filename}" + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || { + echo "Failed to create a temporary file in $path" + return 1 + } if command_available "curl"; then # -o must appear before -- so it is parsed as an option, not an extra URL. if curl -fL -o "$tmp" -- "${download_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -745,7 +482,7 @@ function download() { progress_opt=(-q --show-progress) fi if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${download_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index c886f6f67e..45102ece24 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -15,970 +15,306 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# test-check-port.sh — Unit tests for check_port() in util.sh -# -# Strategy: each test runs check_port in a subshell that overrides -# command_available() to control which probe branch is taken, and -# overrides the tool functions (ss, netstat, timeout) to control -# what they return — no real network connections needed. -# -# check_port calls `exit 1` when the port is in use, so the subshell -# exits 1; it returns normally (exit 0) when the port is free. + +# Contract tests for the startup port preflight in bin/util.sh. # -# Usage: ./test-check-port.sh [path-to-hugegraph-static-dir] -# path-to-hugegraph-static-dir: directory containing bin/util.sh -# Defaults to current directory. -# In CI: $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static +# The preflight is best effort: the server's own bind is authoritative. These +# tests pin the three-state contract (busy / free / unknown) rather than the +# internals of any one probe. -set -uo pipefail -# -u: fail on undefined variables (catches typos in test assertions) -# -o pipefail: pipeline exit status is the last non-zero component -# shellcheck disable=SC1090,SC1091 # UTIL_SH / PD_UTIL_SH sourced dynamically at runtime +set -u -STATIC_DIR="${1:-$(pwd)}" +STATIC_DIR="${1:-hugegraph-server/hugegraph-dist/src/assembly/static}" UTIL_SH="$STATIC_DIR/bin/util.sh" -REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" -PD_UTIL_SH="$REPO_ROOT/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh" -STORE_UTIL_SH="$REPO_ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh" - -PASS=0 -FAIL=0 -ERRORS=() - -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' - -pass() { echo -e "${GREEN} PASS${NC} $1"; PASS=$((PASS + 1)); } -fail() { echo -e "${RED} FAIL${NC} $1"; ERRORS+=("$1"); FAIL=$((FAIL + 1)); } -section() { echo ""; echo "── $1 ──"; } - -# timeout mock used by ss/netstat test cases. It passes getent through so -# normalize_addr can canonicalise numeric IPv6, and pretends every other -# probe (the /dev/tcp fallback) failed, so free-port assertions are not -# influenced by real network state. -timeout() { - if [[ "${2:-}" == "getent" ]]; then - shift - "$@" 2>/dev/null - else - return 1 - fi -} - -echo "" -echo "check_port() unit test suite" -echo "util.sh: $UTIL_SH" -echo "" - if [[ ! -f "$UTIL_SH" ]]; then - echo -e "${RED}ERROR:${NC} $UTIL_SH not found." - echo " Pass the HugeGraph static assembly dir as \$1" - exit 1 + echo "SKIP: util.sh not found at $UTIL_SH" + exit 0 fi -# ── ss branch ───────────────────────────────────────────────────────────────── - -section "ss branch — IPv4" - -( - # shellcheck source=/dev/null - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: IPv4 port occupied → exit 1" \ - || fail "ss: IPv4 port occupied → expected exit 1, got 0" - -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 0.0.0.0:9090 0.0.0.0:*"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: IPv4 port free → exit 0" \ - || fail "ss: IPv4 port free → expected exit 0, got 1" - -section "ss branch — IPv6 URL with scheme (http://[::1]:8080)" - -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: http://[::1]:8080 occupied → exit 1" \ - || fail "ss: http://[::1]:8080 occupied → expected exit 1, got 0" +# shellcheck source=/dev/null +source "$UTIL_SH" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: http://[::1]:8080 free → exit 0" \ - || fail "ss: http://[::1]:8080 free → expected exit 0, got 1" +# Sections run in subshells so their command overrides stay isolated, which +# means results have to be tallied through a file rather than a variable. +RESULTS=$(mktemp) -section "ss branch — IPv6 URL without scheme ([::1]:8080)" +pass() { + echo " PASS $1" + echo "P" >> "$RESULTS" +} -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } - check_port "[::1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: [::1]:8080 (no scheme) occupied → exit 1" \ - || fail "ss: [::1]:8080 (no scheme) occupied → expected exit 1, got 0" +fail() { + echo " FAIL $1" + echo " $2" + echo "F" >> "$RESULTS" +} -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } - check_port "[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: [::1]:8080 (no scheme) free → exit 0" \ - || fail "ss: [::1]:8080 (no scheme) free → expected exit 0, got 1" +expect() { + # expect + if [[ "$2" == "$3" ]]; then + pass "$1" + else + fail "$1" "expected '$2', got '$3'" + fi +} -section "ss branch — wildcard 0.0.0.0" +echo "" +echo "check_port contract tests ($UTIL_SH)" +# -------------------------------------------------------------------------- +echo "" +echo "1. URL to port" +# "|", where SKIP means "no usable port, preflight is skipped". +url_cases=( + 'http://127.0.0.1:8080|8080' + 'HTTP://127.0.0.1:8080|8080' + 'http://127.0.0.1|80' + 'https://127.0.0.1|443' + '127.0.0.1:8080|8080' + 'http://127.0.0.1:8080/path:9090|8080' + 'http://127.0.0.1:8080?probe=x|8080' + 'http://127.0.0.1:8080#frag|8080' + 'http://[::1]:8080|8080' + '[::1]:8080|8080' + 'https://[::1]|443' + 'http://127.0.0.1:08080|8080' + '::1:8080|SKIP' + 'http://127.0.0.1:abc|SKIP' + 'http://127.0.0.1:0|SKIP' + 'http://127.0.0.1:70000|SKIP' + '127.0.0.1|SKIP' +) +for case in "${url_cases[@]}"; do + url="${case%|*}" + want="${case##*|}" + got=$(parse_port_from_url " $url " 2>/dev/null) || got="SKIP" + expect "$url" "$want" "$got" +done + +# -------------------------------------------------------------------------- +echo "" +echo "2. Linux: ss busy / free / failure" ( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } - check_port "http://0.0.0.0:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: 0.0.0.0:8080 occupied → exit 1" \ - || fail "ss: 0.0.0.0:8080 occupied → expected exit 1, got 0" + uname() { echo "Linux"; } + command_available() { [[ "$1" == "ss" ]]; } + SS_OUT="" + SS_RC=0 + ss() { printf '%s' "$SS_OUT"; return "$SS_RC"; } -section "ss branch — wildcard ::" + SS_OUT='LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:*' + expect "listener on 8080 is busy" "busy" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } - check_port "http://[::]:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: [::]:8080 occupied → exit 1" \ - || fail "ss: [::]:8080 occupied → expected exit 1, got 0" + SS_OUT='LISTEN 0 4096 0.0.0.0:22 0.0.0.0:*' + expect "no listener on 8080 is free" "free" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } - check_port "http://[::]:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: [::]:8080 free → exit 0" \ - || fail "ss: [::]:8080 free → expected exit 0, got 1" + # The port must come from the local-address field, not an IPv6 hextet. + SS_OUT='LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*' + expect "IPv6 hextet 8080 is not the port" "free" "$(port_listen_state 8080)" -# ── netstat branch ──────────────────────────────────────────────────────────── + SS_OUT='LISTEN 0 128 [::]:8080 [::]:*' + expect "IPv6 wildcard listener is busy" "busy" "$(port_listen_state 8080)" -section "netstat branch — Linux format (-ltn), occupied" + # A tool that runs but fails proves nothing. + SS_OUT='' + SS_RC=1 + expect "ss failure is unknown" "unknown" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "netstat -ltn: port 8080 occupied → exit 1" \ - || fail "netstat -ltn: port 8080 occupied → expected exit 1, got 0" + # Success with an unusable table proves nothing either. + SS_RC=0 + SS_OUT='' + expect "ss empty output is unknown" "unknown" "$(port_listen_state 8080)" -section "netstat branch — Linux format (-ltn), free" + SS_OUT='some diagnostic banner' + expect "ss unparseable output is unknown" "unknown" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { echo "tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN"; } - check_port "http://127.0.0.1:8080" ) -[[ $? -eq 0 ]] \ - && pass "netstat -ltn: port 8080 free → exit 0" \ - || fail "netstat -ltn: port 8080 free → expected exit 0, got 1" -section "netstat branch — BSD/macOS fallback (-an), occupied" - -# Simulate netstat that produces no output for -ltn (Linux flag unsupported) -# but outputs BSD-format lines for -an +# -------------------------------------------------------------------------- +echo "" +echo "3. macOS/BSD: LISTEN versus ESTABLISHED" ( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { - if [[ "$1" == "-ltn" ]]; then - return 1 # flag not supported on BSD - fi - echo "tcp4 0 0 *.8080 *.* LISTEN" - } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "netstat -an BSD: port 8080 occupied → exit 1" \ - || fail "netstat -an BSD: port 8080 occupied → expected exit 1, got 0" - -section "netstat branch — IP octet false-positive guard" + uname() { echo "Darwin"; } + command_available() { [[ "$1" == "netstat" ]]; } + NS_OUT="" + netstat() { printf '%s' "$NS_OUT"; } -# Port 80 check; netstat output contains 192.168.80.1:443 -# The .80 in the IP address must NOT match port 80 -( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { echo "tcp 0 0 192.168.80.1:443 0.0.0.0:* LISTEN"; } - check_port "http://127.0.0.1:80" -) -[[ $? -eq 0 ]] \ - && pass "netstat: IP octet .80 does not false-positive for port 80 → exit 0" \ - || fail "netstat: IP octet .80 false-positived for port 80 → expected exit 0, got 1" + NS_OUT='Active Internet connections (including servers) +Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp4 0 0 *.8080 *.* LISTEN' + expect "BSD LISTEN on 8080 is busy" "busy" "$(port_listen_state 8080)" -section "ss branch — host dot-escaping guard" + # An outbound connection to :443 is not a local listener on 443. This is + # the false positive that blocked startup on macOS. + NS_OUT='Active Internet connections (including servers) +Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp4 0 0 192.168.1.3.60320 44.195.16.138.443 ESTABLISHED +tcp4 0 0 *.22 *.* LISTEN' + expect "BSD ESTABLISHED peer :443 is free" "free" "$(port_listen_state 443)" -# Host 127.0.0.1 must be matched literally: a listener whose address merely -# matches the pattern with '.' as a regex wildcard (127a0b0c1) must NOT count -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 127a0b0c1:8080 0.0.0.0:*"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: unescaped-dot lookalike 127a0b0c1 does not false-positive → exit 0" \ - || fail "ss: unescaped-dot lookalike 127a0b0c1 false-positived → expected exit 0, got 1" + # A socket lingering in TIME_WAIT holds the port in its *local* address but + # is not a listener, so only the connection state can tell them apart. + NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp4 0 0 127.0.0.1.8080 127.0.0.1.51000 TIME_WAIT +tcp4 0 0 *.22 *.* LISTEN' + expect "BSD TIME_WAIT on local 8080 is free" "free" "$(port_listen_state 8080)" -# ── /dev/tcp fallback branch ────────────────────────────────────────────────── + NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp6 0 0 ::1.8080 *.* LISTEN' + expect "BSD IPv6 LISTEN is busy" "busy" "$(port_listen_state 8080)" -section "/dev/tcp fallback — timeout available, port occupied" + NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state)' + expect "BSD header only is unknown" "unknown" "$(port_listen_state 8080)" -# timeout exits 0 → connection succeeded → port in use -( - source "$UTIL_SH" - command_available() { [[ "$1" == "timeout" ]]; } - timeout() { - # Assert correct invocation: timeout 1 bash -c SCRIPT _ HOST PORT - [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ - || { echo "timeout mock: unexpected argv: $*"; return 2; } - return 0 - } - check_port "http://127.0.0.1:8080" ) -[[ $? -eq 1 ]] \ - && pass "/dev/tcp+timeout: connection succeeded (exit 0) → port occupied → exit 1" \ - || fail "/dev/tcp+timeout: connection succeeded → expected exit 1, got 0" -section "/dev/tcp fallback — timeout available, port free" - -# timeout exits 1 → connection refused → port free -( - source "$UTIL_SH" - command_available() { [[ "$1" == "timeout" ]]; } - timeout() { - [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ - || { echo "timeout mock: unexpected argv: $*"; return 2; } - return 1 - } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "/dev/tcp+timeout: connection refused (exit 1) → port free → exit 0" \ - || fail "/dev/tcp+timeout: connection refused → expected exit 0, got 1" - -section "/dev/tcp fallback — real loopback (ephemeral port)" - -# Start Python server on ephemeral port 0, capture actual port from child stdout +# -------------------------------------------------------------------------- +echo "" +echo "4. Unknown never blocks startup" ( - source "$UTIL_SH" - command_available() { [[ "$1" == "timeout" ]]; } - # Mock timeout: on hosts without timeout (macOS), run the probe directly. - # The probe args are: timeout 1 bash -c SCRIPT _ HOST PORT - timeout() { - [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ - || { echo "timeout mock: unexpected argv: $*" >&2; return 2; } - # Run the probe with a 2-second hard deadline via background + watchdog - bash -c "$4" "$5" "$6" "$7" 2>/dev/null & - local probe_pid=$! - (sleep 2; kill -9 "$probe_pid" 2>/dev/null) & - local watchdog_pid=$! - wait "$probe_pid" 2>/dev/null - local rc=$? - kill -9 "$watchdog_pid" 2>/dev/null - wait "$watchdog_pid" 2>/dev/null - return $rc - } - # Use a temp file to capture the bound port from child - port_file=$(mktemp) - trap 'rm -f "$port_file"; [[ -n "${PY_PID:-}" ]] && { kill -9 "$PY_PID" 2>/dev/null; wait "$PY_PID" 2>/dev/null; }' EXIT - - # Start Python server that prints the bound port to stdout - python3 -c " -import socket -s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -s.bind(('127.0.0.1', 0)) -s.listen(1) # actually listen so /dev/tcp can connect -port = s.getsockname()[1] -print(port, flush=True) -import time -time.sleep(30) # keep server alive -" > "$port_file" 2>/dev/null & - PY_PID=$! - # Poll for port file up to ~4s (Python cold-start can exceed 0.5s on busy CI) - bound_port="" - for _ in $(seq 1 40); do - bound_port=$(head -1 "$port_file" 2>/dev/null) - [[ -n "$bound_port" ]] && break - kill -0 $PY_PID 2>/dev/null || break - sleep 0.1 - done - if [[ -z "$bound_port" || ! "$bound_port" =~ ^[0-9]+$ ]]; then - echo "SKIP: failed to get bound port" - kill -9 $PY_PID 2>/dev/null || true - exit 77 - fi - # Verify child is alive - if ! kill -0 $PY_PID 2>/dev/null; then - echo "SKIP: Python child died" - exit 77 - fi - check_port "http://127.0.0.1:$bound_port" -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP /dev/tcp real ephemeral port (setup failed)" -elif [[ $_rc -eq 1 ]]; then - pass "/dev/tcp real ephemeral port → detects occupation → exit 1" -else - fail "/dev/tcp real ephemeral port → expected exit 1, got $_rc" -fi + uname() { echo "Linux"; } + command_available() { return 1; } -section "/dev/tcp fallback — no-timeout watchdog kills stuck probe" + expect "no probe tool is unknown" "unknown" "$(port_listen_state 8080)" -# run_with_deadline "sleep 5" 2 should return in ~2s, not ~5s. -# Proves the watchdog actually kills a stuck child, not just that the code path is taken. -( - source "$UTIL_SH" - start=$(date +%s) - run_with_deadline 'sleep 5' 2 + # check_port must warn and let the server perform the authoritative bind. + err=$( (check_port "http://127.0.0.1:8080") 2>&1 >/dev/null ) rc=$? - elapsed=$(($(date +%s) - start)) - # Must complete well before the 5s sleep (2s deadline + overhead) and return non-zero (killed) - if [[ $rc -ne 0 && $elapsed -le 4 ]]; then - exit 0 + expect "unknown exits 0" "0" "$rc" + if [[ "$err" == *"could not determine"* ]]; then + pass "unknown warns on stderr" else - echo "run_with_deadline: rc=$rc elapsed=${elapsed}s (expected rc≠0, ≤4s)" >&2 - exit 1 - fi -) -[[ $? -eq 0 ]] \ - && pass "/dev/tcp no-timeout watchdog: sleep 5 killed in ≤4s" \ - || fail "/dev/tcp no-timeout watchdog: watchdog did not kill in time" - -# ── host/port conflict semantics ────────────────────────────────────────────── - -section "Host/port conflict semantics" - -# Configured host 192.168.1.50, but only 127.0.0.1 is listening -> free -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } - check_port "http://192.168.1.50:8080" -) -[[ $? -eq 0 ]] \ - && pass "specific IP configured vs loopback listener → free → exit 0" \ - || fail "specific IP configured vs loopback listener → expected free, got occupied" - -# Configured host 192.168.1.50, 0.0.0.0 is listening -> occupied -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } - check_port "http://192.168.1.50:8080" -) -[[ $? -eq 1 ]] \ - && pass "specific IP configured vs wildcard listener → occupied → exit 1" \ - || fail "specific IP configured vs wildcard listener → expected occupied, got free" - -# Specific IPv4 host should not conflict with IPv6 wildcard listener -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 [::]:8080 [::]:*"; return 0; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "specific IPv4 host vs IPv6 wildcard listener → free → exit 0" \ - || fail "specific IPv4 host vs IPv6 wildcard listener → expected free, got occupied" - -# Specific IPv6 host should not conflict with IPv4 wildcard listener -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "specific IPv6 host vs IPv4 wildcard listener → free → exit 0" \ - || fail "specific IPv6 host vs IPv4 wildcard listener → expected free, got occupied" - -# ── tool failure fallthrough ────────────────────────────────────────────────── - -section "Tool failure fallthrough" - -# ss fails, netstat succeeds and finds occupied port -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } - ss() { return 1; } # ss execution fails - netstat() { - if [[ "$1" == "-ltn" ]]; then echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; return 0; fi - return 1 - } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss exits 1 → fallthrough to netstat → occupied → exit 1" \ - || fail "ss exits 1 → fallthrough to netstat → expected occupied, got free" - -# both ss and netstat fail, /dev/tcp timeout path must still detect occupation -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } - ss() { return 1; } - netstat() { return 1; } - timeout() { return 0; } # /dev/tcp connect succeeds - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss/netstat both fail → /dev/tcp timeout path occupied → exit 1" \ - || fail "ss/netstat both fail → expected /dev/tcp fallback occupation, got free" - -# ── hostname and URL normalization ──────────────────────────────────────────── - -section "Hostname collision detection" - -# localhost should resolve to numeric addresses before ss matching -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } - timeout() { shift; "$@" 2>/dev/null; } # pass-through for mocked getent - getent() { - if [[ "$1" == "hosts" || "$1" == "ahosts" ]] && [[ "$2" == "localhost" ]]; then - echo "127.0.0.1" - return 0 - fi - return 1 - } - ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } - check_port "http://localhost:8080" -) -[[ $? -eq 1 ]] \ - && pass "hostname localhost resolves to 127.0.0.1 and detects occupation → exit 1" \ - || fail "hostname localhost occupation was not detected via resolved address" - -# unresolved hostnames should skip ss/netstat text matching and use /dev/tcp fallback -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } - getent() { return 2; } # resolution fails - ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } - timeout() { return 0; } # fallback connect succeeds - check_port "http://unresolved.localdomain:8080" -) -[[ $? -eq 1 ]] \ - && pass "unresolved hostname falls through to /dev/tcp fallback and detects occupation" \ - || fail "unresolved hostname should use /dev/tcp fallback" - -section "URL normalization and IPv6 hextet guard" - -# check_port should trim URL whitespace before parsing host/port -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } - check_port " http://127.0.0.1:8080 " -) -[[ $? -eq 1 ]] \ - && pass "leading/trailing URL whitespace is normalized → occupied detected" \ - || fail "URL whitespace normalization failed to detect occupied port" - -# Ensure ":8080" inside an IPv6 hextet does not match listener port 8080 -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - timeout() { - if [[ "$2" == "getent" ]]; then - shift - "$@" 2>/dev/null - else - return 1 - fi - } - ss() { echo "tcp LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "IPv6 hextet containing 8080 does not false-positive port 8080" \ - || fail "IPv6 hextet false-positive: expected free, got occupied" - -# ── IPv6 address canonicalisation ───────────────────────────────────────────── - -section "IPv6 address canonicalisation" - -# getent mock that normalises the two spellings of loopback to ::1 -getent_canonical_loopback() { - if [[ "$1" == "ahosts" ]] && [[ "$2" == "::1" || "$2" == "0:0:0:0:0:0:0:1" ]]; then - echo "::1" - return 0 + fail "unknown warns on stderr" "stderr was: $err" fi - return 2 -} - -# Host ::1, listener expanded 0:0:0:0:0:0:0:1 -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - getent() { getent_canonical_loopback "$@"; } - ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:8080 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "IPv6 host [::1] matches expanded listener [0:0:0:0:0:0:0:1]:8080" \ - || fail "IPv6 compressed vs expanded spelling did not match" - -# Host expanded 0:0:0:0:0:0:0:1, listener ::1 (mixed case) -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - getent() { getent_canonical_loopback "$@"; } - ss() { echo "tcp LISTEN 0 128 [::1]:8080 [::]:*"; } - check_port "http://[0:0:0:0:0:0:0:1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "IPv6 expanded host matches compressed listener" \ - || fail "IPv6 expanded vs compressed spelling did not match" - -# Host ::1, listener expanded on a different port → free -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - timeout() { - if [[ "$2" == "getent" ]]; then - shift - "$@" 2>/dev/null - else - return 1 - fi - } - getent() { getent_canonical_loopback "$@"; } - ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:9090 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "IPv6 canonicalisation does not false-positive on different port" \ - || fail "IPv6 canonicalisation matched the wrong port" - -# ── edge cases ──────────────────────────────────────────────────────────────── - -section "Edge cases" - -# Empty URL → port string is empty → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "" -) -[[ $? -eq 0 ]] \ - && pass "empty URL → exit 0 (graceful)" \ - || fail "empty URL → expected exit 0, got 1" - -# Port out of range (> 65535) → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "http://127.0.0.1:99999" -) -[[ $? -eq 0 ]] \ - && pass "port 99999 (out of range) → exit 0 (graceful)" \ - || fail "port 99999 (out of range) → expected exit 0, got 1" -# Port 0 (invalid) → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "http://127.0.0.1:0" -) -[[ $? -eq 0 ]] \ - && pass "port 0 (invalid) → exit 0 (graceful)" \ - || fail "port 0 (invalid) → expected exit 0, got 1" - -# Non-numeric port → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "127.0.0.1:abc" ) -[[ $? -eq 0 ]] \ - && pass "non-numeric port → exit 0 (graceful)" \ - || fail "non-numeric port → expected exit 0, got 1" - -# ── download() fallback ─────────────────────────────────────────────────────── - -section "download() atomic curl path (pd)" - -( - if [[ ! -f "$PD_UTIL_SH" ]]; then - exit 77 - fi - - source "$PD_UTIL_SH" - - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT - - command_available() { - if [[ "$1" == "wget" ]]; then return 1; fi - if [[ "$1" == "curl" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } - CURL_CALLED=0 - curl() { - # Expected: curl -fL -o -- - if [[ "$1" == "-fL" && "$2" == "-o" && \ - "$4" == "--" && \ - "$5" == "https://example.com/some/path/file.tar.gz" ]]; then - CURL_CALLED=1 - # The temp name must be hidden and contain the PID. - if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "curl mock: unexpected output path: $3" - return 1 - fi - touch "$3" - return 0 - fi - echo "curl mock called with unexpected args: $*" - return 1 - } +# -------------------------------------------------------------------------- +echo "" +echo "5. Real listener on an ephemeral port" +if command -v python3 >/dev/null 2>&1; then + PORT_FILE=$(mktemp) + python3 -c ' +import socket, sys, time +s = socket.socket() +s.bind(("127.0.0.1", 0)) +s.listen(1) +print(s.getsockname()[1]) +sys.stdout.flush() +time.sleep(30) +' > "$PORT_FILE" & + PY_PID=$! + trap 'kill "$PY_PID" 2>/dev/null; rm -f "$PORT_FILE"' EXIT - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? + BOUND="" + for _ in 1 2 3 4 5 6 7 8 9 10; do + BOUND=$(head -1 "$PORT_FILE" 2>/dev/null) + [[ -n "$BOUND" ]] && break + sleep 0.5 + done - if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 + if [[ -z "$BOUND" ]] || ! kill -0 "$PY_PID" 2>/dev/null; then + fail "listener bound an ephemeral port" "child did not report a port" else - echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + pass "listener bound an ephemeral port ($BOUND)" + expect "bound port is busy" "busy" "$(port_listen_state "$BOUND")" + (check_port "http://127.0.0.1:$BOUND" >/dev/null 2>&1) + expect "check_port exits 1 on the bound port" "1" "$?" + + # A port that was bound and released must not read as busy. + FREE=$(python3 -c 'import socket +s = socket.socket() +s.bind(("127.0.0.1", 0)) +p = s.getsockname()[1] +s.close() +print(p)') + expect "released port is not busy" "0" "$([[ "$(port_listen_state "$FREE")" == "busy" ]] && echo 1 || echo 0)" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic curl path: pd util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (PD) writes to temp and renames on success" + + kill "$PY_PID" 2>/dev/null + wait "$PY_PID" 2>/dev/null + rm -f "$PORT_FILE" + trap - EXIT else - fail "download() (PD) atomic curl path failed" + echo " SKIP real listener test: python3 not available" fi -section "download() atomic curl path (store)" - +# -------------------------------------------------------------------------- +echo "" +echo "6. download() rename failure is reported" ( - if [[ ! -f "$STORE_UTIL_SH" ]]; then - exit 77 - fi + WORK=$(mktemp -d) + trap 'rm -rf "$WORK"' EXIT - source "$STORE_UTIL_SH" - - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT - - command_available() { - if [[ "$1" == "wget" ]]; then return 1; fi - if [[ "$1" == "curl" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } - - CURL_CALLED=0 + command_available() { [[ "$1" == "curl" ]]; } curl() { - if [[ "$1" == "-fL" && "$2" == "-o" && \ - "$4" == "--" && \ - "$5" == "https://example.com/some/path/file.tar.gz" ]]; then - CURL_CALLED=1 - if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "curl mock: unexpected output path: $3" - return 1 - fi - touch "$3" - return 0 - fi - echo "curl mock called with unexpected args: $*" - return 1 + # Write to the -o destination so a temp file exists to rename. + local out="" + while [[ $# -gt 0 ]]; do + [[ "$1" == "-o" ]] && { out="$2"; shift; } + shift + done + echo "payload" > "$out" + return 0 } + # Simulate a rename that fails (read-only target, cross-device, ...). + mv() { return 1; } - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? - - if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 + if download "$WORK" "https://example.com/pkg.tar.gz"; then + fail "download reports rename failure" "download returned 0 after mv failed" else - echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + pass "download reports rename failure" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic curl path: store util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (Store) writes to temp and renames on success" -else - fail "download() (Store) atomic curl path failed" -fi -section "download() atomic wget path (pd)" - -( - if [[ ! -f "$PD_UTIL_SH" ]]; then - exit 77 + if [[ -e "$WORK/pkg.tar.gz" ]]; then + fail "failed rename leaves no destination" "$WORK/pkg.tar.gz exists" + else + pass "failed rename leaves no destination" fi - source "$PD_UTIL_SH" - - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT - - command_available() { - if [[ "$1" == "curl" ]]; then return 1; fi - if [[ "$1" == "wget" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } + leftover=$(find "$WORK" -name '.pkg.tar.gz.*' 2>/dev/null | wc -l | tr -d ' ') + expect "failed rename cleans its temp file" "0" "$leftover" - WGET_CALLED=0 - wget() { - if [[ "$1" == "--help" ]]; then - # wget --help runs in a pipeline subshell, so side-effect counters - # cannot be observed from the parent. Only the stdout matters here. - echo "--show-progress" - return 0 - fi - # Expected: wget -q --show-progress -O -- - if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ - "$5" == "--" && \ - "$6" == "https://example.com/some/path/file.tar.gz" ]]; then - WGET_CALLED=1 - if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "wget mock: unexpected output path: $4" - return 1 - fi - touch "$4" - return 0 - fi - echo "wget mock called with unexpected args: $*" - return 1 - } - - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? - - if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 - else - echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 - fi ) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic wget path: pd util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (PD) wget writes to temp and renames on success" -else - fail "download() (PD) atomic wget path failed" -fi - -section "download() atomic wget path (store)" +# -------------------------------------------------------------------------- +echo "" +echo "7. Startup probe is bounded" ( - if [[ ! -f "$STORE_UTIL_SH" ]]; then - exit 77 - fi + WORK=$(mktemp -d) + cd "$WORK" || exit 1 + trap 'cd /; rm -rf "$WORK"' EXIT - source "$STORE_UTIL_SH" + ARGS_FILE="$WORK/curl-args" + curl() { echo "$*" >> "$ARGS_FILE"; echo "000"; return 28; } + process_status() { return 0; } + ps() { return 0; } - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + wait_for_startup "$$" "test-server" "http://127.0.0.1:1" 1 >/dev/null 2>&1 - command_available() { - if [[ "$1" == "curl" ]]; then return 1; fi - if [[ "$1" == "wget" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } - - WGET_CALLED=0 - wget() { - if [[ "$1" == "--help" ]]; then - # wget --help runs in a pipeline subshell, so side-effect counters - # cannot be observed from the parent. Only the stdout matters here. - echo "--show-progress" - return 0 - fi - # Expected: wget -q --show-progress -O -- - if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ - "$5" == "--" && \ - "$6" == "https://example.com/some/path/file.tar.gz" ]]; then - WGET_CALLED=1 - if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "wget mock: unexpected output path: $4" - return 1 - fi - touch "$4" - return 0 - fi - echo "wget mock called with unexpected args: $*" - return 1 - } - - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? - - if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 + if grep -q -- "--max-time" "$ARGS_FILE" 2>/dev/null; then + pass "startup probe passes --max-time" else - echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + fail "startup probe passes --max-time" "args were: $(cat "$ARGS_FILE" 2>/dev/null)" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic wget path: store util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (Store) wget writes to temp and renames on success" -else - fail "download() (Store) atomic wget path failed" -fi - -section "download() curl failure cleanup (pd)" - -# curl failure must clean up the partial file and never leave a poisoned artifact -( - if [[ ! -f "$PD_UTIL_SH" ]]; then exit 77; fi - source "$PD_UTIL_SH" - - OUTDIR=$(mktemp -d) - trap 'rm -rf "$OUTDIR"' EXIT - - command_available() { - if [[ "$1" == "wget" ]]; then return 1; fi - if [[ "$1" == "curl" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - mkdir() { command mkdir -p "$2"; return 0; } - PARTIAL="" - curl() { - if [[ "$1" == "-fL" && "$2" == "-o" && \ - "$4" == "--" && \ - "$5" == "https://example.com/some/path/file.tar.gz" ]]; then - PARTIAL="$3" - # Simulate a partial/corrupted transfer: write something then fail. - echo "partial" > "$PARTIAL" - return 1 - fi - return 1 - } - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - rc=$? - - if [[ $rc -ne 0 && ! -f "$OUTDIR/file.tar.gz" && \ - ( -z "$PARTIAL" || ! -f "$PARTIAL" ) ]]; then - exit 0 + if grep -q -- "--connect-timeout" "$ARGS_FILE" 2>/dev/null; then + pass "startup probe passes --connect-timeout" else - echo "rc=$rc PARTIAL=$PARTIAL DEST=$([[ -f $OUTDIR/file.tar.gz ]] && echo exists || echo missing)" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + fail "startup probe passes --connect-timeout" "args were: $(cat "$ARGS_FILE" 2>/dev/null)" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() curl failure cleanup: pd util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() curl failure cleans temp and does not poison destination" -else - fail "download() curl failure did not clean partial file" -fi -# ── summary ─────────────────────────────────────────────────────────────────── +) +# -------------------------------------------------------------------------- echo "" -echo "════════════════════════════════" -echo -e " Results: ${GREEN}$PASS passed${NC} ${RED}$FAIL failed${NC}" -echo "════════════════════════════════" - -if [[ $FAIL -gt 0 ]]; then - echo "" - echo "Failed tests:" - for err in ${ERRORS[@]+"${ERRORS[@]}"}; do - echo -e " ${RED}✗${NC} $err" - done +echo "----------------------------------------" +# grep -c prints 0 and exits 1 when there are no matches; the count is what +# matters here, so the exit status is deliberately ignored. +PASSED=$(grep -c '^P' "$RESULTS" 2>/dev/null) +FAILED=$(grep -c '^F' "$RESULTS" 2>/dev/null) +rm -f "$RESULTS" +echo "passed: $PASSED failed: $FAILED" +if [[ "$FAILED" -gt 0 ]]; then + exit 1 fi - -echo "" -[[ $FAIL -eq 0 ]] && exit 0 || exit 1 +exit 0 diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 5e68d4cbb3..b4011e2e47 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -279,15 +279,19 @@ function download() { } fi - local filename + local filename tmp filename=$(basename "${link_url%%[?#]*}") - local tmp="${path}/.${filename}.tmp.$$" local dest="${path}/${filename}" + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || { + echo "Failed to create a temporary file in $path" + return 1 + } if command_available "curl"; then # -o must appear before -- so it is parsed as an option, not an extra URL. if curl -fL -o "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -298,7 +302,7 @@ function download() { progress_opt=(-q --show-progress) fi if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -329,7 +333,9 @@ download_and_verify() { fi echo "Downloading $filepath..." - local tmp="${filepath}.tmp.$$" + local tmp + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${filepath}.XXXXXX") || return 1 if curl -fL -o "$tmp" -- "$url"; then actual_md5=$(md5sum -- "$tmp" | awk '{ print $1 }') if [[ "$actual_md5" != "$expected_md5" ]]; then @@ -337,7 +343,9 @@ download_and_verify() { rm -f -- "$tmp" return 1 fi - mv -f -- "$tmp" "$filepath" + # A failed rename must not report success: callers export LD_PRELOAD + # from $filepath and would otherwise use a library that never installed. + mv -f -- "$tmp" "$filepath" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 From ea81ad764556f43e7992324e48c472c58913adfc Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 27 Jul 2026 20:22:03 +0530 Subject: [PATCH 4/4] docs(bin): record the port-preflight tradeoffs, and fix userinfo parsing Converging on port-only detection removed capability on purpose. Mark each gap in place with a namespaced TODO so it can be found and picked up later rather than rediscovered: - port-only matching ignores the listener's address, so a listener on one local address reports the port busy even when the server would bind a different one. This is the old `lsof -i :PORT` behaviour and fails safe, but it can refuse a bind that would have succeeded. - a host with genuinely zero LISTEN sockets is indistinguishable from a restricted or unparseable table; both report unknown and warn. - with neither ss nor netstat present there is no probe left, so the check is permanently unknown. A dependency-free fallback needs a bounded connect, which is exactly what was removed here. - unbracketed IPv6 and a scheme-less value with no port are skipped rather than guessed. - wait_for_startup overshoot is now bounded but not zero: the loop still sleeps between probes before re-reading the clock. - the Linux and BSD detection branches are mock-driven, so only the host's own branch runs against a real kernel on any one runner. Also fix a real parsing defect found while auditing those gaps: a URL carrying userinfo (http://user:pass@host:port) was matched by the unbracketed-IPv6 check, so it skipped the preflight and printed a misleading warning about bracket notation. Strip userinfo from the authority before that test, and cover the userinfo forms in the suite. Suite: 44 passed, 0 failed. --- .../src/assembly/static/bin/util.sh | 31 +++++++++++++++++++ .../src/assembly/travis/test-check-port.sh | 9 ++++++ 2 files changed, 40 insertions(+) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index c794bd6d2e..6f9e62d0f7 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -141,6 +141,11 @@ function parse_port_from_url() { local authority="${rest%%[/?#]*}" [[ -z "$authority" ]] && return 1 + # Drop any userinfo prefix; its colon would otherwise look like an + # unbracketed IPv6 separator. + authority="${authority##*@}" + [[ -z "$authority" ]] && return 1 + local port="" if [[ "$authority" =~ ^\[[^]]*\](:([0-9]+))?$ ]]; then # Bracketed IPv6, with or without a port: [::1] or [::1]:8080 @@ -148,6 +153,9 @@ function parse_port_from_url() { elif [[ "$authority" == *:*:* ]]; then # Unbracketed IPv6 is ambiguous: in "::1:8080" the trailing group may # be a port or another hextet. Refuse to guess. + # TODO(check_port): no preflight runs at all for this form. If + # ServerOptions ever guarantees a normalized bracketed value here, this + # branch can resolve the port instead of skipping the check. echo "WARN: ambiguous IPv6 authority '$authority' in server URL;" \ "use bracket notation such as [::1]:8080." >&2 return 1 @@ -156,6 +164,9 @@ function parse_port_from_url() { fi # Fall back to the scheme's default port. + # TODO(check_port): a scheme-less value with no explicit port (e.g. plain + # "127.0.0.1") has no derivable port, so it is skipped rather than guessed. + # Reading the configured default from ServerOptions would close this gap. if [[ -z "$port" ]]; then case "$scheme" in http) port="80" ;; @@ -184,6 +195,18 @@ function parse_port_from_url() { # unrelated outbound connection to the same port number is never mistaken for # a local listener. A tool that is missing, fails, or yields no recognisable # listener row reports "unknown" rather than "free". +# +# TODO(check_port): port-only matching ignores the listener's address, so a +# listener bound to one local address (127.0.0.1:8080) reports the port busy +# even when the server would bind a different one (192.168.1.5:8080). This is +# deliberate - it is what `lsof -i :PORT` did, and it fails safe - but it can +# refuse a bind that would have succeeded. If that is reported in practice, +# revisit by comparing the local-address column instead of only its port. +# +# TODO(check_port): a host with genuinely zero LISTEN sockets is indistinguish- +# able from a restricted or unparseable table, so both report "unknown" and +# warn on every start. Distinguishing them needs a positive signal that the +# table was readable (e.g. an exit status ss/netstat do not currently give). function port_listen_state() { local port="$1" local out @@ -229,6 +252,11 @@ function port_listen_state() { fi fi + # TODO(check_port): with neither ss nor netstat present (some minimal + # container images ship neither) there is no probe left, so the preflight + # is permanently "unknown" and never detects a busy port. A dependency- + # free fallback would need a bounded connect, which was deliberately + # removed here; adding one back means re-solving the hang this PR fixes. echo "unknown" } @@ -313,6 +341,9 @@ function wait_for_startup() { # Bound each probe by the time left in the overall deadline: without # --max-time a single blackholed request blocks past ${timeout_s}s. + # TODO(wait_for_startup): overshoot is now bounded but not zero - the + # loop still sleeps 2s after a probe and only then re-reads the clock, + # so the total can exceed ${timeout_s}s by roughly one sleep interval. local remain_s=$((stop_s - now_s)) [ "$remain_s" -lt 1 ] && remain_s=1 local connect_s=$((remain_s < 5 ? remain_s : 5)) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 45102ece24..21bfbeb37c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -21,6 +21,12 @@ # The preflight is best effort: the server's own bind is authoritative. These # tests pin the three-state contract (busy / free / unknown) rather than the # internals of any one probe. +# +# TODO(test-check-port): the Linux and BSD detection branches are both driven +# by mocked tool output, so on any one runner only the host's own branch is +# ever exercised against a real kernel. The single real-listener case covers +# whichever OS the job runs on. Closing this needs the suite to run on both +# a Linux and a macOS runner, which CI already does for the server job. set -u @@ -75,7 +81,10 @@ url_cases=( 'http://127.0.0.1:8080/path:9090|8080' 'http://127.0.0.1:8080?probe=x|8080' 'http://127.0.0.1:8080#frag|8080' + 'http://user:pass@127.0.0.1:8080|8080' + 'http://user@127.0.0.1:8080|8080' 'http://[::1]:8080|8080' + 'http://user:pass@[::1]:8080|8080' '[::1]:8080|8080' 'https://[::1]|443' 'http://127.0.0.1:08080|8080'