diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 0d6c54b51c..e332de705a 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -70,17 +70,30 @@ jobs: run: | mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + - name: Run check_port unit tests + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + # Validates ss/netstat//dev/tcp replacement for lsof + $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + - name: Check startup test prerequisites id: server-preflight if: ${{ env.BACKEND == 'rocksdb' }} run: | - for tool in lsof crontab curl java; do + # Note: lsof removed from prerequisites (replaced with ss/netstat//dev/tcp in check_port). + # This job always runs on Linux (ubuntu-22.04), which uses fuser for port cleanup. + for tool in crontab curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" exit 0 fi done + if ! command -v fuser >/dev/null 2>&1; then + echo "can_run=false" >> "$GITHUB_OUTPUT" + echo "skip_reason=missing tool: fuser (required for Linux port cleanup)" >> "$GITHUB_OUTPUT" + exit 0 + fi echo "can_run=true" >> "$GITHUB_OUTPUT" - name: Run start-hugegraph.sh foreground mode tests @@ -178,6 +191,11 @@ jobs: run: | mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + - name: Run check_port unit tests + # Validates ss/netstat//dev/tcp replacement for lsof + run: | + $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + - name: Run RocksDB core test run: | $TRAVIS_DIR/run-core-test.sh $BACKEND diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index 0b7ad0f0f0..e80255672e 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -20,20 +20,21 @@ # TODO: consider reuse it with server-dist module (almost same as it) function command_available() { local cmd=$1 - if [ $(command -v $cmd >/dev/null 2>&1) ]; then - return 1 - else + if command -v "$cmd" >/dev/null 2>&1; then return 0 fi + return 1 } # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." - property_name=$(echo $2 | sed 's/\./\\\./g') - cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 + property_name=$(echo "$2" | sed 's/\./\\\./g') + cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 } function write_property() { @@ -56,7 +57,7 @@ function parse_yaml() { local version=$2 local module=$3 - cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{ pre=""; for(i=1; i<=NF; ) { if(match($i, /version/)) { @@ -72,27 +73,19 @@ function parse_yaml() { } function process_num() { - num=`ps -ef | grep $1 | grep -v grep | wc -l` - return $num + local num + num=$(ps -ef | grep "$1" | grep -v grep | wc -l) + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { - pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` - return $pid -} - -# check the port of rest server is occupied -function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 - fi - lsof -i :$port >/dev/null - if [ $? -eq 0 ]; then - echo "The port $port has already been used" - exit 1 - fi + local pid + pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') + echo "$pid" + return 0 } function crontab_append() { @@ -130,13 +123,14 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=`date '+%s'` - local stop_s=$(( $now_s + $timeout_s )) + local now_s + now_s=$(date '+%s') + local stop_s=$(( now_s + timeout_s )) local status echo -n "Connecting to $server_name ($server_url)" - while [ $now_s -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -144,14 +138,14 @@ function wait_for_startup() { return 1 fi - status=`curl -o /dev/null -s -k -w %{http_code} $server_url` - if [[ $status -eq 200 || $status -eq 401 ]]; then + status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url") + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "The operation timed out when attempting to connect to $server_url" >&2 @@ -160,22 +154,28 @@ function wait_for_startup() { function free_memory() { local free="" - local os=`uname` + local os=$(uname) if [ "$os" == "Linux" ]; then - local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` - local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` - local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}') + local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}') + local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}') if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then echo "Failed to get free memory" exit 1 fi - free=`expr $mem_free + $mem_buffer + $mem_cached` - free=`expr $free / 1024` + free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") + free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_available=`expr $pages_free + $pages_inactive` - free=`expr $pages_available \* 4096 / 1024 / 1024` + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") + free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" exit 1 @@ -187,8 +187,8 @@ function calc_xmx() { local min_mem=$1 local max_mem=$2 # Get machine available memory - local free=`free_memory` - local half_free=$[free/2] + local free=$(free_memory) + local half_free=$((free/2)) local xmx=$min_mem if [[ "$free" -lt "$min_mem" ]]; then @@ -236,47 +236,74 @@ function ensure_path_writable() { } function get_ip() { - local os=`uname` + local os=$(uname) local loopback="127.0.0.1" local ip="" case $os in Linux) if command_available "ifconfig"; then - ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}') elif command_available "ip"; then - ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}') else ip=$loopback fi ;; FreeBSD|OpenBSD|Darwin) if command_available "ifconfig"; then - ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}') else ip=$loopback fi ;; SunOS) if command_available "ifconfig"; then - ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ') else ip=$loopback fi ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local link_url=$2 - if command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget ${link_url} -P ${path} $progress_opt - elif command_available "curl"; then - curl ${link_url} -o ${path}/${link_url} + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + local filename + filename=$(basename "${link_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + + if command_available "curl"; then + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi + elif command_available "wget"; then + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required wget or curl but they are unavailable" exit 1 @@ -289,19 +316,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d ${path}/${dir} ]; then - if [ ! -f ${path}/${tar} ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download ${path} ${link} - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi @@ -316,7 +341,7 @@ function wait_for_shutdown() { local pid="$2" local timeout_s="$3" - local now_s=`date '+%s'` + local now_s=$(date '+%s') local stop_s=$(( $now_s + $timeout_s )) echo -n "Killing $process_name(pid $pid)" >&2 @@ -328,7 +353,7 @@ function wait_for_shutdown() { return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 return 1 @@ -357,7 +382,7 @@ function kill_process() { return 0 fi - case "`uname`" in + case "$(uname)" in CYGWIN*) taskkill /F /PID "$pid" ;; *) kill "$pid" ;; esac diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 91c0c3efbf..687e5d2001 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -18,11 +18,10 @@ function command_available() { local cmd=$1 - if [[ -x "$(command -v "$cmd")" ]]; then + if command -v "$cmd" >/dev/null 2>&1; then return 0 - else - return 1 fi + return 1 } function configure_riscv64_libatomic() { @@ -67,6 +66,8 @@ function configure_riscv64_libatomic() { # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." property_name=$(echo "$2" | sed 's/\./\\\./g') @@ -109,27 +110,417 @@ function parse_yaml() { } function process_num() { + local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - return "$num" + # Return 0 when no process, 1 when one or more. Using $num directly as an + # exit code would truncate values > 255, so treat this as a boolean result. + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { + local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - return "$pid" + echo "$pid" + return 0 +} + +# Run a command with a hard deadline via background watchdog. +# Returns the command's exit code if it finishes in time. +# If the deadline expires, the command is killed (exit code reflects signal). +# Works without the timeout command — uses sleep + kill -9 pattern. +function run_with_deadline() { + local cmd="$1" + local deadline="$2" + shift 2 + + bash -c "$cmd" bash "$@" & + local child_pid=$! + ( + local sleep_pid="" + cleanup_watchdog() { + [[ -n "$sleep_pid" ]] && kill -9 "$sleep_pid" 2>/dev/null + } + # Kill any direct children of the target PID (e.g. a spawned sleep) + # in case killing the wrapper left them orphaned. + kill_children() { + local child + for child in $(pgrep -P "$1" 2>/dev/null); do + kill -9 "$child" 2>/dev/null || true + done + } + trap 'cleanup_watchdog' EXIT TERM + sleep "$deadline" & sleep_pid=$! + wait "$sleep_pid" 2>/dev/null + if kill -0 "$child_pid" 2>/dev/null; then + kill -9 "$child_pid" 2>/dev/null + kill_children "$child_pid" + fi + ) 2>/dev/null & + local watchdog_pid=$! + + wait "$child_pid" 2>/dev/null + local rc=$? + # Kill the watchdog with SIGTERM so its EXIT trap reaps the sleep child. + kill -TERM "$watchdog_pid" 2>/dev/null || true + wait "$watchdog_pid" 2>/dev/null || true + return $rc +} + + +# Normalize an IP address to a canonical string for comparison. +# - IPv4 addresses are returned unchanged. +# - IPv6 addresses are stripped of brackets/zone scope, lowercased and +# compressed to the canonical textual form (via getent ahosts). +# - IPv4-mapped IPv6 (::ffff:1.2.3.4) is collapsed to the IPv4 form. +# - Hostnames are returned unchanged; callers should resolve them first. +function normalize_addr() { + local addr="$1" + + # Strip brackets + if [[ "$addr" =~ ^\[.*\]$ ]]; then + addr="${addr#\[}" + addr="${addr%\]}" + fi + + # Drop IPv6 zone scope (e.g. 127.0.0.53%lo, fe80::1%eth0) + addr="${addr%%\%*}" + + # IPv4-mapped IPv6 -> IPv4 so a bound IPv4-mapped socket is compared + # against an IPv4-configured address. + if [[ "$addr" =~ ^::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + echo "${BASH_REMATCH[1]}" + return + fi + + # Plain IPv4 + if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$addr" + return + fi + + # IPv6: ask glibc for the canonical compressed form + if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + local norm + if command_available "timeout"; then + norm=$(timeout 2 getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') + else + norm=$(getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') + fi + if [[ -n "$norm" ]]; then + echo "$norm" + return + fi + # macOS/BSD fallback: python3's socket module can canonicalise + # numeric IPv6 even when getent is unavailable. + if command_available "python3"; then + norm=$(python3 -c 'import socket, sys; print(socket.inet_ntop(socket.AF_INET6, socket.inet_pton(socket.AF_INET6, sys.argv[1])))' "$addr" 2>/dev/null) + if [[ -n "$norm" ]]; then + echo "$norm" + return + fi + fi + # No canonicaliser available: at least normalise hex case so ss/netstat + # lowercase output still matches an uppercase configured address. + echo "$addr" | tr '[:upper:]' '[:lower:]' + return + fi + + # Fall back to the cleaned original + echo "$addr" } -# check the port of rest server is occupied +# check whether the REST server port is occupied function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 + local url="$1" + local host + local port + + # Strip leading/trailing whitespace from URL (handles whitespace from ServerOptions) + url="${url#"${url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + + # Extract authority: strip scheme and stop at the first /, ? or #. + local authority + authority="${url#*://}" + authority="${authority%%[/?#]*}" + + # Extract host and port from authority. + if [[ "$authority" =~ ^\[([^\]]*)\]:([0-9]+)$ ]]; then + # IPv6 with port: [::1]:8080 + host="${BASH_REMATCH[1]}" + port="${BASH_REMATCH[2]}" + elif [[ "$authority" =~ ^\[([^\]]*)\]$ ]]; then + # IPv6 without port: [::1] + host="${BASH_REMATCH[1]}" + port="" + elif [[ "$authority" =~ :([0-9]+)$ ]]; then + # IPv4 or hostname with port: 127.0.0.1:8080, localhost:8080 + port="${BASH_REMATCH[1]}" + host="${authority%:*}" + else + # No explicit port in authority + host="$authority" + port="" fi - lsof -i :"$port" >/dev/null - if [ $? -eq 0 ]; then + + # Handle default ports from scheme when no explicit port is given + if [[ -z "$port" ]]; then + if [[ "$url" == https://* ]]; then + port="443" + elif [[ "$url" == http://* ]]; then + port="80" + else + return 0 + fi + fi + + # Validate port as a decimal number + if ! [[ "$port" =~ ^[0-9]+$ ]]; then + return 0 + fi + port=$((10#$port)) + if (( port < 1 || port > 65535 )); then + return 0 + fi + + # Strip any leading/trailing whitespace from host + host="${host#"${host%%[![:space:]]*}"}" + host="${host%"${host##*[![:space:]]}"}" + + local norm_host + norm_host=$(normalize_addr "$host") + + # Determine the address family of the configured host so we only treat + # same-family wildcard listeners as conflicts (IPv4 vs IPv6 sockets are + # separate unless explicitly dual-stacked). + local host_family="" + if [[ "$norm_host" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + host_family="ipv4" + elif [[ "$norm_host" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + host_family="ipv6" + else + host_family="hostname" + fi + + local in_use=0 + local port_checked=0 + + # Wildcard binds are detected by looking for any listener on the port. + # Specific hosts are matched against the local address; if the listener + # is a same-family wildcard (0.0.0.0 / :: / *), that is a conflict too. + local is_wildcard=0 + if [[ -z "$host" || "$host" == "0.0.0.0" || "$host" == "::" || "$host" == "*" ]]; then + is_wildcard=1 + fi + + # Build the list of acceptable normalized addresses for a specific host. + # Hostnames must be resolved to numeric IPs first because ss/netstat + # output is always numeric. + local candidate_addrs="" + if [[ $is_wildcard -eq 0 ]]; then + if [[ "$host_family" == "ipv4" || "$host_family" == "ipv6" ]]; then + # Already numeric + candidate_addrs="$norm_host" + else + # Hostname: resolve with deadline + if command_available "getent" && command_available "timeout"; then + candidate_addrs=$(timeout 2 getent ahosts "$host" 2>/dev/null | awk '{print $1}') + elif command_available "dscacheutil" && command_available "timeout"; then + candidate_addrs=$(timeout 2 dscacheutil -q host -a name "$host" 2>/dev/null \ + | awk '/ip_address:/{print $2}') + fi + fi + fi + + # Helper: scan a line of listener-table output and return 0 if it matches + # the configured host/port. Sets 'matched_token' to 1 when it evaluates a + # token that we can trust, so callers know whether "no match" is reliable. + local out line listener_addr norm_listener token matched_token + + # Returns true if the listener address is a wildcard on the same family + # as the configured host (or any family, if the host is a wildcard). + _check_port_wildcard_conflicts() { + local wl_addr="$1" + if [[ $is_wildcard -eq 1 ]]; then + return 0 + fi + # BSD netstat prints *. for an any-family wildcard. + if [[ "$wl_addr" == "*" ]]; then + return 0 + fi + if [[ "$wl_addr" == "0.0.0.0" ]]; then + [[ "$host_family" == "ipv4" ]] && return 0 + # A hostname that resolved to IPv4 can also bind an IPv4 wildcard + if [[ "$host_family" == "hostname" ]]; then + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + return 0 + fi + done <<< "$candidate_addrs" + fi + fi + if [[ "$wl_addr" == "::" ]]; then + [[ "$host_family" == "ipv6" ]] && return 0 + if [[ "$host_family" == "hostname" ]]; then + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + return 0 + fi + done <<< "$candidate_addrs" + fi + fi + return 1 + } + + _check_port_match_listener_line() { + local out_line="$1" + # Skip empty or whitespace-only lines before read -a to avoid an empty + # array expansion under set -u on Bash 3.2. + if [[ -z "${out_line//[[:space:]]/}" ]]; then + return 1 + fi + local -a tokens + read -r -a tokens <<< "$out_line" + for token in ${tokens[@]+"${tokens[@]}"}; do + # We only care about the "local address:port" token. For the + # common tools this is the first token that ends with the + # target port after ':' or '.' (peer addresses use :* on Linux + # and *.* on BSD, so they never match a numeric port). + if [[ "$token" =~ ^(.*):(${port})$ ]]; then + listener_addr="${BASH_REMATCH[1]}" + elif [[ "$token" =~ ^(.*)\.(${port})$ ]]; then + listener_addr="${BASH_REMATCH[1]}" + else + continue + fi + + # Wildcard host: any listener on this port is a conflict. + if [[ $is_wildcard -eq 1 ]]; then + matched_token=1 + return 0 + fi + + # No resolved candidates (hostname resolution failed/unsupported): + # we cannot reliably compare against the numeric listener table. + if [[ -z "$candidate_addrs" ]]; then + continue + fi + + matched_token=1 + norm_listener=$(normalize_addr "$listener_addr") + + # A same-family wildcard listener on this port conflicts with any + # specific host of that family. + if _check_port_wildcard_conflicts "$norm_listener"; then + return 0 + fi + + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$norm_listener" == "$(normalize_addr "$addr")" ]]; then + return 0 + fi + done <<< "$candidate_addrs" + done + return 1 + } + + if command_available "ss"; then + matched_token=0 + if out=$(ss -ltn 2>/dev/null); then + while IFS= read -r line; do + if _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + # ss -ltn succeeded. We can trust a non-match when we have a + # numeric host or resolved addresses (or a wildcard); otherwise + # fall through to /dev/tcp for unresolved hostnames. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + fi + fi + + if [[ $in_use -eq 0 && $port_checked -eq 0 ]] && command_available "netstat"; then + matched_token=0 + if out=$(netstat -ltn 2>/dev/null) && echo "$out" | grep -qi "listen"; then + while IFS= read -r line; do + if _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + # netstat -ltn output is the complete Linux listener table. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + elif out=$(netstat -an 2>/dev/null) && [[ -n "$out" ]]; then + local old_nocasematch + old_nocasematch=$(shopt -p nocasematch 2>/dev/null || true) + shopt -s nocasematch 2>/dev/null || true + while IFS= read -r line; do + if [[ "$line" == *listen* ]] && _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + eval "$old_nocasematch" 2>/dev/null || true + # netstat -an output is the complete BSD listener table. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + fi + fi + + if [[ $in_use -eq 0 && $port_checked -eq 0 ]]; then + # Probe the actual configured endpoint(s) with a short deadline. + local probe_addrs="$candidate_addrs" + if [[ -z "$probe_addrs" ]]; then + # Could not resolve (or wildcard with only loopback probe needed) + if [[ $is_wildcard -eq 1 ]]; then + probe_addrs="127.0.0.1 ::1" + else + probe_addrs="$host" + fi + fi + + local addr + for addr in $probe_addrs; do + # /dev/tcp needs unbracketed, normalized addresses + addr=$(normalize_addr "$addr") + [[ -z "$addr" ]] && continue + if command_available "timeout"; then + if timeout 1 bash -c ': >/dev/tcp/"$1"/"$2"' _ "$addr" "$port" 2>/dev/null; then + in_use=1 + break + fi + else + if run_with_deadline ': >/dev/tcp/"$1"/"$2" 2>/dev/null' 2 "$addr" "$port"; then + in_use=1 + break + fi + fi + done + fi + + local _rc=0 + if [[ "$in_use" -eq 1 ]]; then echo "The port $port has already been used" - exit 1 + _rc=1 fi + unset -f _check_port_wildcard_conflicts _check_port_match_listener_line || true + [[ $_rc -eq 1 ]] && exit 1 + return 0 } function crontab_append() { @@ -167,14 +558,15 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=$(date '+%s') + local now_s + now_s=$(date '+%s') local stop_s=$((now_s + timeout_s)) local status local error_file_name="startup_error.txt" echo -n "Connecting to $server_name ($server_url)" - while [ "$now_s" -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -186,7 +578,7 @@ function wait_for_startup() { fi status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") - if [[ $status -eq 200 || $status -eq 401 ]]; then + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" if [ -e "$error_file_name" ]; then @@ -219,9 +611,15 @@ function free_memory() { free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - local pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - local pages_available=$(expr "$pages_free" + "$pages_inactive") + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" @@ -312,23 +710,46 @@ function get_ip() { ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local download_url=$2 + + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + # Strip query/fragment so the on-disk name matches the server-side artifact. + local filename + filename=$(basename "${download_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + if command_available "curl"; then - if [ ! -d "$path" ]; then - mkdir -p "$path" || { - echo "Failed to create directory: $path" - exit 1 - } + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${download_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 fi - curl -L "${download_url}" -o "${path}/$(basename "${download_url}")" elif command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget "${download_url}" -P "${path}" $progress_opt + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${download_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required curl or wget but they are unavailable" exit 1 @@ -341,19 +762,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d "${path}"/"${dir}" ]; then - if [ ! -f "${path}"/"${tar}" ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download "${path}" "${link}" - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar zxvf "${path}"/"${tar}" -C "${path}" >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh new file mode 100755 index 0000000000..c886f6f67e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -0,0 +1,984 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# test-check-port.sh — Unit tests for check_port() in util.sh +# +# Strategy: each test runs check_port in a subshell that overrides +# command_available() to control which probe branch is taken, and +# overrides the tool functions (ss, netstat, timeout) to control +# what they return — no real network connections needed. +# +# check_port calls `exit 1` when the port is in use, so the subshell +# exits 1; it returns normally (exit 0) when the port is free. +# +# Usage: ./test-check-port.sh [path-to-hugegraph-static-dir] +# path-to-hugegraph-static-dir: directory containing bin/util.sh +# Defaults to current directory. +# In CI: $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + +set -uo pipefail +# -u: fail on undefined variables (catches typos in test assertions) +# -o pipefail: pipeline exit status is the last non-zero component +# shellcheck disable=SC1090,SC1091 # UTIL_SH / PD_UTIL_SH sourced dynamically at runtime + +STATIC_DIR="${1:-$(pwd)}" +UTIL_SH="$STATIC_DIR/bin/util.sh" + +REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +PD_UTIL_SH="$REPO_ROOT/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh" +STORE_UTIL_SH="$REPO_ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh" + +PASS=0 +FAIL=0 +ERRORS=() + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +pass() { echo -e "${GREEN} PASS${NC} $1"; PASS=$((PASS + 1)); } +fail() { echo -e "${RED} FAIL${NC} $1"; ERRORS+=("$1"); FAIL=$((FAIL + 1)); } +section() { echo ""; echo "── $1 ──"; } + +# timeout mock used by ss/netstat test cases. It passes getent through so +# normalize_addr can canonicalise numeric IPv6, and pretends every other +# probe (the /dev/tcp fallback) failed, so free-port assertions are not +# influenced by real network state. +timeout() { + if [[ "${2:-}" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi +} + +echo "" +echo "check_port() unit test suite" +echo "util.sh: $UTIL_SH" +echo "" + +if [[ ! -f "$UTIL_SH" ]]; then + echo -e "${RED}ERROR:${NC} $UTIL_SH not found." + echo " Pass the HugeGraph static assembly dir as \$1" + exit 1 +fi + +# ── ss branch ───────────────────────────────────────────────────────────────── + +section "ss branch — IPv4" + +( + # shellcheck source=/dev/null + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: IPv4 port occupied → exit 1" \ + || fail "ss: IPv4 port occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:9090 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: IPv4 port free → exit 0" \ + || fail "ss: IPv4 port free → expected exit 0, got 1" + +section "ss branch — IPv6 URL with scheme (http://[::1]:8080)" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: http://[::1]:8080 occupied → exit 1" \ + || fail "ss: http://[::1]:8080 occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: http://[::1]:8080 free → exit 0" \ + || fail "ss: http://[::1]:8080 free → expected exit 0, got 1" + +section "ss branch — IPv6 URL without scheme ([::1]:8080)" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: [::1]:8080 (no scheme) occupied → exit 1" \ + || fail "ss: [::1]:8080 (no scheme) occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: [::1]:8080 (no scheme) free → exit 0" \ + || fail "ss: [::1]:8080 (no scheme) free → expected exit 0, got 1" + +section "ss branch — wildcard 0.0.0.0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } + check_port "http://0.0.0.0:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: 0.0.0.0:8080 occupied → exit 1" \ + || fail "ss: 0.0.0.0:8080 occupied → expected exit 1, got 0" + +section "ss branch — wildcard ::" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "http://[::]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: [::]:8080 occupied → exit 1" \ + || fail "ss: [::]:8080 occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "http://[::]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: [::]:8080 free → exit 0" \ + || fail "ss: [::]:8080 free → expected exit 0, got 1" + +# ── netstat branch ──────────────────────────────────────────────────────────── + +section "netstat branch — Linux format (-ltn), occupied" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "netstat -ltn: port 8080 occupied → exit 1" \ + || fail "netstat -ltn: port 8080 occupied → expected exit 1, got 0" + +section "netstat branch — Linux format (-ltn), free" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "netstat -ltn: port 8080 free → exit 0" \ + || fail "netstat -ltn: port 8080 free → expected exit 0, got 1" + +section "netstat branch — BSD/macOS fallback (-an), occupied" + +# Simulate netstat that produces no output for -ltn (Linux flag unsupported) +# but outputs BSD-format lines for -an +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { + if [[ "$1" == "-ltn" ]]; then + return 1 # flag not supported on BSD + fi + echo "tcp4 0 0 *.8080 *.* LISTEN" + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "netstat -an BSD: port 8080 occupied → exit 1" \ + || fail "netstat -an BSD: port 8080 occupied → expected exit 1, got 0" + +section "netstat branch — IP octet false-positive guard" + +# Port 80 check; netstat output contains 192.168.80.1:443 +# The .80 in the IP address must NOT match port 80 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 192.168.80.1:443 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:80" +) +[[ $? -eq 0 ]] \ + && pass "netstat: IP octet .80 does not false-positive for port 80 → exit 0" \ + || fail "netstat: IP octet .80 false-positived for port 80 → expected exit 0, got 1" + +section "ss branch — host dot-escaping guard" + +# Host 127.0.0.1 must be matched literally: a listener whose address merely +# matches the pattern with '.' as a regex wildcard (127a0b0c1) must NOT count +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 127a0b0c1:8080 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: unescaped-dot lookalike 127a0b0c1 does not false-positive → exit 0" \ + || fail "ss: unescaped-dot lookalike 127a0b0c1 false-positived → expected exit 0, got 1" + +# ── /dev/tcp fallback branch ────────────────────────────────────────────────── + +section "/dev/tcp fallback — timeout available, port occupied" + +# timeout exits 0 → connection succeeded → port in use +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + timeout() { + # Assert correct invocation: timeout 1 bash -c SCRIPT _ HOST PORT + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*"; return 2; } + return 0 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "/dev/tcp+timeout: connection succeeded (exit 0) → port occupied → exit 1" \ + || fail "/dev/tcp+timeout: connection succeeded → expected exit 1, got 0" + +section "/dev/tcp fallback — timeout available, port free" + +# timeout exits 1 → connection refused → port free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + timeout() { + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*"; return 2; } + return 1 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "/dev/tcp+timeout: connection refused (exit 1) → port free → exit 0" \ + || fail "/dev/tcp+timeout: connection refused → expected exit 0, got 1" + +section "/dev/tcp fallback — real loopback (ephemeral port)" + +# Start Python server on ephemeral port 0, capture actual port from child stdout +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + # Mock timeout: on hosts without timeout (macOS), run the probe directly. + # The probe args are: timeout 1 bash -c SCRIPT _ HOST PORT + timeout() { + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*" >&2; return 2; } + # Run the probe with a 2-second hard deadline via background + watchdog + bash -c "$4" "$5" "$6" "$7" 2>/dev/null & + local probe_pid=$! + (sleep 2; kill -9 "$probe_pid" 2>/dev/null) & + local watchdog_pid=$! + wait "$probe_pid" 2>/dev/null + local rc=$? + kill -9 "$watchdog_pid" 2>/dev/null + wait "$watchdog_pid" 2>/dev/null + return $rc + } + # Use a temp file to capture the bound port from child + port_file=$(mktemp) + trap 'rm -f "$port_file"; [[ -n "${PY_PID:-}" ]] && { kill -9 "$PY_PID" 2>/dev/null; wait "$PY_PID" 2>/dev/null; }' EXIT + + # Start Python server that prints the bound port to stdout + python3 -c " +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(('127.0.0.1', 0)) +s.listen(1) # actually listen so /dev/tcp can connect +port = s.getsockname()[1] +print(port, flush=True) +import time +time.sleep(30) # keep server alive +" > "$port_file" 2>/dev/null & + PY_PID=$! + # Poll for port file up to ~4s (Python cold-start can exceed 0.5s on busy CI) + bound_port="" + for _ in $(seq 1 40); do + bound_port=$(head -1 "$port_file" 2>/dev/null) + [[ -n "$bound_port" ]] && break + kill -0 $PY_PID 2>/dev/null || break + sleep 0.1 + done + if [[ -z "$bound_port" || ! "$bound_port" =~ ^[0-9]+$ ]]; then + echo "SKIP: failed to get bound port" + kill -9 $PY_PID 2>/dev/null || true + exit 77 + fi + # Verify child is alive + if ! kill -0 $PY_PID 2>/dev/null; then + echo "SKIP: Python child died" + exit 77 + fi + check_port "http://127.0.0.1:$bound_port" +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP /dev/tcp real ephemeral port (setup failed)" +elif [[ $_rc -eq 1 ]]; then + pass "/dev/tcp real ephemeral port → detects occupation → exit 1" +else + fail "/dev/tcp real ephemeral port → expected exit 1, got $_rc" +fi + +section "/dev/tcp fallback — no-timeout watchdog kills stuck probe" + +# run_with_deadline "sleep 5" 2 should return in ~2s, not ~5s. +# Proves the watchdog actually kills a stuck child, not just that the code path is taken. +( + source "$UTIL_SH" + start=$(date +%s) + run_with_deadline 'sleep 5' 2 + rc=$? + elapsed=$(($(date +%s) - start)) + # Must complete well before the 5s sleep (2s deadline + overhead) and return non-zero (killed) + if [[ $rc -ne 0 && $elapsed -le 4 ]]; then + exit 0 + else + echo "run_with_deadline: rc=$rc elapsed=${elapsed}s (expected rc≠0, ≤4s)" >&2 + exit 1 + fi +) +[[ $? -eq 0 ]] \ + && pass "/dev/tcp no-timeout watchdog: sleep 5 killed in ≤4s" \ + || fail "/dev/tcp no-timeout watchdog: watchdog did not kill in time" + +# ── host/port conflict semantics ────────────────────────────────────────────── + +section "Host/port conflict semantics" + +# Configured host 192.168.1.50, but only 127.0.0.1 is listening -> free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } + check_port "http://192.168.1.50:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IP configured vs loopback listener → free → exit 0" \ + || fail "specific IP configured vs loopback listener → expected free, got occupied" + +# Configured host 192.168.1.50, 0.0.0.0 is listening -> occupied +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } + check_port "http://192.168.1.50:8080" +) +[[ $? -eq 1 ]] \ + && pass "specific IP configured vs wildcard listener → occupied → exit 1" \ + || fail "specific IP configured vs wildcard listener → expected occupied, got free" + +# Specific IPv4 host should not conflict with IPv6 wildcard listener +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 [::]:8080 [::]:*"; return 0; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IPv4 host vs IPv6 wildcard listener → free → exit 0" \ + || fail "specific IPv4 host vs IPv6 wildcard listener → expected free, got occupied" + +# Specific IPv6 host should not conflict with IPv4 wildcard listener +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IPv6 host vs IPv4 wildcard listener → free → exit 0" \ + || fail "specific IPv6 host vs IPv4 wildcard listener → expected free, got occupied" + +# ── tool failure fallthrough ────────────────────────────────────────────────── + +section "Tool failure fallthrough" + +# ss fails, netstat succeeds and finds occupied port +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } + ss() { return 1; } # ss execution fails + netstat() { + if [[ "$1" == "-ltn" ]]; then echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; return 0; fi + return 1 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss exits 1 → fallthrough to netstat → occupied → exit 1" \ + || fail "ss exits 1 → fallthrough to netstat → expected occupied, got free" + +# both ss and netstat fail, /dev/tcp timeout path must still detect occupation +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } + ss() { return 1; } + netstat() { return 1; } + timeout() { return 0; } # /dev/tcp connect succeeds + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss/netstat both fail → /dev/tcp timeout path occupied → exit 1" \ + || fail "ss/netstat both fail → expected /dev/tcp fallback occupation, got free" + +# ── hostname and URL normalization ──────────────────────────────────────────── + +section "Hostname collision detection" + +# localhost should resolve to numeric addresses before ss matching +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } + timeout() { shift; "$@" 2>/dev/null; } # pass-through for mocked getent + getent() { + if [[ "$1" == "hosts" || "$1" == "ahosts" ]] && [[ "$2" == "localhost" ]]; then + echo "127.0.0.1" + return 0 + fi + return 1 + } + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } + check_port "http://localhost:8080" +) +[[ $? -eq 1 ]] \ + && pass "hostname localhost resolves to 127.0.0.1 and detects occupation → exit 1" \ + || fail "hostname localhost occupation was not detected via resolved address" + +# unresolved hostnames should skip ss/netstat text matching and use /dev/tcp fallback +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } + getent() { return 2; } # resolution fails + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } + timeout() { return 0; } # fallback connect succeeds + check_port "http://unresolved.localdomain:8080" +) +[[ $? -eq 1 ]] \ + && pass "unresolved hostname falls through to /dev/tcp fallback and detects occupation" \ + || fail "unresolved hostname should use /dev/tcp fallback" + +section "URL normalization and IPv6 hextet guard" + +# check_port should trim URL whitespace before parsing host/port +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } + check_port " http://127.0.0.1:8080 " +) +[[ $? -eq 1 ]] \ + && pass "leading/trailing URL whitespace is normalized → occupied detected" \ + || fail "URL whitespace normalization failed to detect occupied port" + +# Ensure ":8080" inside an IPv6 hextet does not match listener port 8080 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + timeout() { + if [[ "$2" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi + } + ss() { echo "tcp LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "IPv6 hextet containing 8080 does not false-positive port 8080" \ + || fail "IPv6 hextet false-positive: expected free, got occupied" + +# ── IPv6 address canonicalisation ───────────────────────────────────────────── + +section "IPv6 address canonicalisation" + +# getent mock that normalises the two spellings of loopback to ::1 +getent_canonical_loopback() { + if [[ "$1" == "ahosts" ]] && [[ "$2" == "::1" || "$2" == "0:0:0:0:0:0:0:1" ]]; then + echo "::1" + return 0 + fi + return 2 +} + +# Host ::1, listener expanded 0:0:0:0:0:0:0:1 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:8080 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "IPv6 host [::1] matches expanded listener [0:0:0:0:0:0:0:1]:8080" \ + || fail "IPv6 compressed vs expanded spelling did not match" + +# Host expanded 0:0:0:0:0:0:0:1, listener ::1 (mixed case) +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [::1]:8080 [::]:*"; } + check_port "http://[0:0:0:0:0:0:0:1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "IPv6 expanded host matches compressed listener" \ + || fail "IPv6 expanded vs compressed spelling did not match" + +# Host ::1, listener expanded on a different port → free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + timeout() { + if [[ "$2" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi + } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "IPv6 canonicalisation does not false-positive on different port" \ + || fail "IPv6 canonicalisation matched the wrong port" + +# ── edge cases ──────────────────────────────────────────────────────────────── + +section "Edge cases" + +# Empty URL → port string is empty → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "" +) +[[ $? -eq 0 ]] \ + && pass "empty URL → exit 0 (graceful)" \ + || fail "empty URL → expected exit 0, got 1" + +# Port out of range (> 65535) → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "http://127.0.0.1:99999" +) +[[ $? -eq 0 ]] \ + && pass "port 99999 (out of range) → exit 0 (graceful)" \ + || fail "port 99999 (out of range) → expected exit 0, got 1" + +# Port 0 (invalid) → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "http://127.0.0.1:0" +) +[[ $? -eq 0 ]] \ + && pass "port 0 (invalid) → exit 0 (graceful)" \ + || fail "port 0 (invalid) → expected exit 0, got 1" + +# Non-numeric port → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "127.0.0.1:abc" +) +[[ $? -eq 0 ]] \ + && pass "non-numeric port → exit 0 (graceful)" \ + || fail "non-numeric port → expected exit 0, got 1" + +# ── download() fallback ─────────────────────────────────────────────────────── + +section "download() atomic curl path (pd)" + +( + if [[ ! -f "$PD_UTIL_SH" ]]; then + exit 77 + fi + + source "$PD_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + CURL_CALLED=0 + curl() { + # Expected: curl -fL -o -- + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + CURL_CALLED=1 + # The temp name must be hidden and contain the PID. + if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "curl mock: unexpected output path: $3" + return 1 + fi + touch "$3" + return 0 + fi + echo "curl mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic curl path: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (PD) writes to temp and renames on success" +else + fail "download() (PD) atomic curl path failed" +fi + +section "download() atomic curl path (store)" + +( + if [[ ! -f "$STORE_UTIL_SH" ]]; then + exit 77 + fi + + source "$STORE_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + CURL_CALLED=0 + curl() { + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + CURL_CALLED=1 + if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "curl mock: unexpected output path: $3" + return 1 + fi + touch "$3" + return 0 + fi + echo "curl mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic curl path: store util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (Store) writes to temp and renames on success" +else + fail "download() (Store) atomic curl path failed" +fi + +section "download() atomic wget path (pd)" + +( + if [[ ! -f "$PD_UTIL_SH" ]]; then + exit 77 + fi + + source "$PD_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "curl" ]]; then return 1; fi + if [[ "$1" == "wget" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + WGET_CALLED=0 + wget() { + if [[ "$1" == "--help" ]]; then + # wget --help runs in a pipeline subshell, so side-effect counters + # cannot be observed from the parent. Only the stdout matters here. + echo "--show-progress" + return 0 + fi + # Expected: wget -q --show-progress -O -- + if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ + "$5" == "--" && \ + "$6" == "https://example.com/some/path/file.tar.gz" ]]; then + WGET_CALLED=1 + if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "wget mock: unexpected output path: $4" + return 1 + fi + touch "$4" + return 0 + fi + echo "wget mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic wget path: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (PD) wget writes to temp and renames on success" +else + fail "download() (PD) atomic wget path failed" +fi + +section "download() atomic wget path (store)" + +( + if [[ ! -f "$STORE_UTIL_SH" ]]; then + exit 77 + fi + + source "$STORE_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "curl" ]]; then return 1; fi + if [[ "$1" == "wget" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + WGET_CALLED=0 + wget() { + if [[ "$1" == "--help" ]]; then + # wget --help runs in a pipeline subshell, so side-effect counters + # cannot be observed from the parent. Only the stdout matters here. + echo "--show-progress" + return 0 + fi + # Expected: wget -q --show-progress -O -- + if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ + "$5" == "--" && \ + "$6" == "https://example.com/some/path/file.tar.gz" ]]; then + WGET_CALLED=1 + if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "wget mock: unexpected output path: $4" + return 1 + fi + touch "$4" + return 0 + fi + echo "wget mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic wget path: store util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (Store) wget writes to temp and renames on success" +else + fail "download() (Store) atomic wget path failed" +fi + +section "download() curl failure cleanup (pd)" + +# curl failure must clean up the partial file and never leave a poisoned artifact +( + if [[ ! -f "$PD_UTIL_SH" ]]; then exit 77; fi + source "$PD_UTIL_SH" + + OUTDIR=$(mktemp -d) + trap 'rm -rf "$OUTDIR"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + mkdir() { command mkdir -p "$2"; return 0; } + PARTIAL="" + curl() { + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + PARTIAL="$3" + # Simulate a partial/corrupted transfer: write something then fail. + echo "partial" > "$PARTIAL" + return 1 + fi + return 1 + } + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + rc=$? + + if [[ $rc -ne 0 && ! -f "$OUTDIR/file.tar.gz" && \ + ( -z "$PARTIAL" || ! -f "$PARTIAL" ) ]]; then + exit 0 + else + echo "rc=$rc PARTIAL=$PARTIAL DEST=$([[ -f $OUTDIR/file.tar.gz ]] && echo exists || echo missing)" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() curl failure cleanup: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() curl failure cleans temp and does not poison destination" +else + fail "download() curl failure did not clean partial file" +fi + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo "" +echo "════════════════════════════════" +echo -e " Results: ${GREEN}$PASS passed${NC} ${RED}$FAIL failed${NC}" +echo "════════════════════════════════" + +if [[ $FAIL -gt 0 ]]; then + echo "" + echo "Failed tests:" + for err in ${ERRORS[@]+"${ERRORS[@]}"}; do + echo -e " ${RED}✗${NC} $err" + done +fi + +echo "" +[[ $FAIL -eq 0 ]] && exit 0 || exit 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh index 260e7f5711..c6926c6dfd 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh @@ -30,6 +30,7 @@ set -uo pipefail PD_ROOT="${1:-$(pwd)}" BIN="$PD_ROOT/bin" START_SCRIPT="$BIN/start-hugegraph-pd.sh" +STOP_SCRIPT="$BIN/stop-hugegraph-pd.sh" PID_FILE="$BIN/pid" PD_URL="http://localhost:8620" STARTUP_WAIT=60 # seconds to wait for PD HTTP to respond @@ -67,14 +68,29 @@ section() { cleanup() { info "Cleaning up..." + "$STOP_SCRIPT" >/dev/null 2>&1 || true if [[ -s "$PID_FILE" ]]; then kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" rm -rf "$PD_ROOT/logs/" - # kill anything still holding the PD port - lsof -ti :8620 | xargs kill -9 2>/dev/null || true - lsof -ti :8686 | xargs kill -9 2>/dev/null || true + # kill anything still holding the PD port (fuser avoids lsof dependency) + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8620 8686; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + fuser -k 8620/tcp 2>/dev/null || true + fuser -k 8686/tcp 2>/dev/null || true + else + echo "Warning: Neither lsof nor fuser available for PD port cleanup" + fi sleep 3 } @@ -129,7 +145,12 @@ if [[ ! -f "$START_SCRIPT" ]]; then exit 1 fi -for tool in lsof curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof curl java" +else + _prereq_tools="fuser curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 7e11da28dd..9ff324ae29 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -51,15 +51,30 @@ section() { echo ""; echo "── $1 ──"; } cleanup() { info "Cleaning up..." + "$STOP_SCRIPT" >/dev/null 2>&1 || true if [[ -s "$PID_FILE" ]]; then kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" rm -rf "$STORE_ROOT/logs/" # kill anything holding Store ports (8520 REST, 8510 raft, 8500 gRPC) - lsof -ti :8520 | xargs kill -9 2>/dev/null || true - lsof -ti :8510 | xargs kill -9 2>/dev/null || true - lsof -ti :8500 | xargs kill -9 2>/dev/null || true + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8520 8510 8500; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + fuser -k 8520/tcp 2>/dev/null || true + fuser -k 8510/tcp 2>/dev/null || true + fuser -k 8500/tcp 2>/dev/null || true + else + echo "Warning: Neither lsof nor fuser available for Store port cleanup" + fi sleep 3 } @@ -110,7 +125,12 @@ if [[ ! -f "$START_SCRIPT" ]]; then exit 1 fi -for tool in lsof curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof curl java" +else + _prereq_tools="fuser curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index 353cab6518..c8e5bfced5 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -76,10 +76,26 @@ cleanup() { kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" - # kill anything still holding server ports so check_port doesn't fail next test - lsof -ti :8080 | xargs kill -9 2>/dev/null || true - lsof -ti :8182 | xargs kill -9 2>/dev/null || true - lsof -ti :8088 | xargs kill -9 2>/dev/null || true + # kill anything still holding server ports (fuser avoids lsof dependency) + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8080 8182 8088; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + # fuser exists and should work on Linux; suppress failures but log if debug needed + fuser -k 8080/tcp 2>/dev/null || true + fuser -k 8182/tcp 2>/dev/null || true + fuser -k 8088/tcp 2>/dev/null || true + else + # Neither lsof nor fuser available; ports may remain occupied + echo "Warning: Neither lsof nor fuser available for port cleanup" + fi sleep 3 crontab -l 2>/dev/null | grep -v monitor-hugegraph | crontab - 2>/dev/null || true } @@ -145,7 +161,12 @@ if [[ ! -f "$STOP_SCRIPT" ]]; then exit 1 fi -for tool in lsof crontab curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof crontab curl java" +else + _prereq_tools="fuser crontab curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 @@ -155,7 +176,8 @@ done # start-monitor.sh requires JAVA_HOME if [[ -z "${JAVA_HOME:-}" ]]; then if command -v /usr/libexec/java_home >/dev/null 2>&1; then - export JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)" + JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)" + export JAVA_HOME fi fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 5a11eeab39..88165e47b6 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -46,8 +46,8 @@ if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$TOP/bin/libjemalloc_aarch64.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" - if download_and_verify $download_url $lib_file $expected_md5; then - export LD_PRELOAD=$lib_file + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + export LD_PRELOAD="$lib_file" else echo "Failed to verify or download $lib_file, skip it" fi @@ -55,8 +55,8 @@ elif [[ $arch == "x86_64" ]]; then lib_file="$TOP/bin/libjemalloc.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" - if download_and_verify $download_url $lib_file $expected_md5; then - export LD_PRELOAD=$lib_file + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + export LD_PRELOAD="$lib_file" else echo "Failed to verify or download $lib_file, skip it" fi @@ -73,7 +73,8 @@ export FILE_LIMITN=1024 #export FILE_LIMITN=1024000 function check_evn_limit() { - local limit_check=$(ulimit -n) + local limit_check + limit_check=$(ulimit -n) if [[ ${limit_check} != "unlimited" && ${limit_check} -lt ${FILE_LIMITN} ]]; then echo -e "${BASH_SOURCE[0]##*/}:${LINENO}:\E[1;32m ulimit -n can open too few maximum file descriptors, need (${FILE_LIMITN})!! \E[0m" return 1 @@ -218,7 +219,7 @@ fi # JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml" #fi -if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then +if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then echo "HugeGraphStoreServer is already running..." exit 0 fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 3b3d660102..5e68d4cbb3 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -19,20 +19,21 @@ function command_available() { local cmd=$1 - if [ `command -v $cmd >/dev/null 2>&1` ]; then - return 1 - else + if command -v "$cmd" >/dev/null 2>&1; then return 0 fi + return 1 } # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." - property_name=`echo $2 | sed 's/\./\\\./g'` - cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 + property_name=$(echo "$2" | sed 's/\./\\\./g') + cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 } function write_property() { @@ -40,7 +41,7 @@ function write_property() { local key=$2 local value=$3 - local os=`uname` + local os=$(uname) case $os in # Note: in mac os should use sed -i '' "xxx" to replace string, # otherwise prompt 'command c expects \ followed by text'. @@ -55,7 +56,7 @@ function parse_yaml() { local version=$2 local module=$3 - cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{ pre=""; for(i=1; i<=NF; ) { if(match($i, /version/)) { @@ -71,27 +72,19 @@ function parse_yaml() { } function process_num() { - num=`ps -ef | grep $1 | grep -v grep | wc -l` - return $num + local num + num=$(ps -ef | grep "$1" | grep -v grep | wc -l) + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { - pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` - return $pid -} - -# check the port of rest server is occupied -function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 - fi - lsof -i :$port >/dev/null - if [ $? -eq 0 ]; then - echo "The port $port has already been used" - exit 1 - fi + local pid + pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') + echo "$pid" + return 0 } function crontab_append() { @@ -129,13 +122,14 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=`date '+%s'` - local stop_s=$(( $now_s + $timeout_s )) + local now_s + now_s=$(date '+%s') + local stop_s=$(( now_s + timeout_s )) local status echo -n "Connecting to $server_name ($server_url)" - while [ $now_s -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -143,14 +137,14 @@ function wait_for_startup() { return 1 fi - status=`curl -o /dev/null -s -k -w %{http_code} $server_url` - if [[ $status -eq 200 || $status -eq 401 ]]; then + status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url") + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "The operation timed out when attempting to connect to $server_url" >&2 @@ -159,22 +153,28 @@ function wait_for_startup() { function free_memory() { local free="" - local os=`uname` + local os=$(uname) if [ "$os" == "Linux" ]; then - local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` - local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` - local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}') + local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}') + local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}') if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then echo "Failed to get free memory" exit 1 fi - free=`expr $mem_free + $mem_buffer + $mem_cached` - free=`expr $free / 1024` + free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") + free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_available=`expr $pages_free + $pages_inactive` - free=`expr $pages_available \* 4096 / 1024 / 1024` + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") + free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" exit 1 @@ -186,8 +186,8 @@ function calc_xmx() { local min_mem=$1 local max_mem=$2 # Get machine available memory - local free=`free_memory` - local half_free=$[free/2] + local free=$(free_memory) + local half_free=$((free/2)) local xmx=$min_mem if [[ "$free" -lt "$min_mem" ]]; then @@ -235,47 +235,74 @@ function ensure_path_writable() { } function get_ip() { - local os=`uname` + local os=$(uname) local loopback="127.0.0.1" local ip="" case $os in Linux) if command_available "ifconfig"; then - ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}') elif command_available "ip"; then - ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}') else ip=$loopback fi ;; FreeBSD|OpenBSD|Darwin) if command_available "ifconfig"; then - ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}') else ip=$loopback fi ;; SunOS) if command_available "ifconfig"; then - ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ') else ip=$loopback fi ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local link_url=$2 - if command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget ${link_url} -P ${path} $progress_opt - elif command_available "curl"; then - curl ${link_url} -o ${path}/${link_url} + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + local filename + filename=$(basename "${link_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + + if command_available "curl"; then + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi + elif command_available "wget"; then + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required wget or curl but they are unavailable" exit 1 @@ -286,14 +313,15 @@ download_and_verify() { local url=$1 local filepath=$2 local expected_md5=$3 + local actual_md5 - if [[ -f $filepath ]]; then + if [[ -f "$filepath" ]]; then echo "File $filepath exists. Verifying MD5 checksum..." - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then + actual_md5=$(md5sum -- "$filepath" | awk '{ print $1 }') + if [[ "$actual_md5" != "$expected_md5" ]]; then echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" echo "Deleting $filepath..." - rm -f $filepath + rm -f -- "$filepath" else echo "MD5 checksum verification succeeded for $filepath." return 0 @@ -301,11 +329,17 @@ download_and_verify() { fi echo "Downloading $filepath..." - curl -L -o $filepath $url - - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + local tmp="${filepath}.tmp.$$" + if curl -fL -o "$tmp" -- "$url"; then + actual_md5=$(md5sum -- "$tmp" | awk '{ print $1 }') + if [[ "$actual_md5" != "$expected_md5" ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + rm -f -- "$tmp" + return 1 + fi + mv -f -- "$tmp" "$filepath" + else + rm -f -- "$tmp" return 1 fi @@ -318,19 +352,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d ${path}/${dir} ]; then - if [ ! -f ${path}/${tar} ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download ${path} ${link} - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi @@ -345,7 +377,7 @@ function wait_for_shutdown() { local pid="$2" local timeout_s="$3" - local now_s=`date '+%s'` + local now_s=$(date '+%s') local stop_s=$(( $now_s + $timeout_s )) echo -n "Killing $process_name(pid $pid)" >&2 @@ -357,7 +389,7 @@ function wait_for_shutdown() { return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 return 1 @@ -386,7 +418,7 @@ function kill_process() { return 0 fi - case "`uname`" in + case "$(uname)" in CYGWIN*) taskkill /F /PID "$pid" ;; *) kill "$pid" ;; esac