Skip to content

Commit 977c3bb

Browse files
adoroszlaichihsuan
andauthored
HDDS-17838. Add Maven Wrapper (#12070)
Co-authored-by: Chi-Hsuan Huang <chihsuan.tw@gmail.com>
1 parent 96caaac commit 977c3bb

26 files changed

Lines changed: 627 additions & 110 deletions

File tree

‎.github/actions/setup-java-with-cache/action.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ runs:
5858
id: cache-key
5959
shell: bash
6060
run: |
61-
echo "value=maven-repo-${{ hashFiles('**/pom.xml', '.mvn/extensions.xml') }}" >> "$GITHUB_OUTPUT"
61+
echo "value=maven-repo-${{ hashFiles('**/pom.xml', '.mvn/extensions.xml', '.mvn/wrapper/maven-wrapper.properties') }}" >> "$GITHUB_OUTPUT"
6262
6363
- name: Remove any leftover Maven repo
6464
id: cleanup
@@ -73,6 +73,7 @@ runs:
7373
path: |
7474
~/.m2/repository/*/*/*
7575
!~/.m2/repository/org/apache/ozone
76+
~/.m2/wrapper
7677
key: ${{ steps.cache-key.outputs.value }}
7778

7879
- name: Exact hit?
@@ -99,6 +100,7 @@ runs:
99100
path: |
100101
~/.m2/repository/*/*/*
101102
!~/.m2/repository/org/apache/ozone
103+
~/.m2/wrapper
102104
key: maven-repo-
103105

104106
- name: Any hit?

‎.github/workflows/check.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ env:
138138
HADOOP_IMAGE: ghcr.io/apache/hadoop
139139
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml -Ddevelocity.user.config=${{ github.workspace }}/.mvn/develocity-asf.xml
140140
MAVEN_OPTS: -Daether.connector.http.connectionMaxTtl=25 -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.intervalMax=30000
141+
MVNW_VERBOSE: true
141142
OZONE_IMAGE: ghcr.io/apache/ozone
142143
OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
143144
OZONE_VOLUME_OWNER: 1000
@@ -230,9 +231,10 @@ jobs:
230231
# TODO remove hard-code in HDDS-17509
231232
if: ${{ inputs.script == 'acceptance' }}
232233
run: |
233-
HADOOP_VERSION=$(mvn help:evaluate -N -Dexpression=hadoop.version -q -DforceStdout -Dscan=false)
234+
source hadoop-ozone/dev-support/checks/_lib.sh
235+
HADOOP_VERSION=$(get_version hadoop)
234236
echo "HADOOP_VERSION=$HADOOP_VERSION" >> $GITHUB_ENV
235-
RANGER_VERSION=$(mvn help:evaluate -N -Dexpression=ranger.version -q -DforceStdout -Dscan=false)
237+
RANGER_VERSION=$(get_version ranger)
236238
echo "RANGER_VERSION=$RANGER_VERSION" >> $GITHUB_ENV
237239
238240
- name: Restore Hadoop

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ env:
4444
# MAVEN_ARGS and MAVEN_OPTS are duplicated in several workflows, please keep in sync
4545
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml -Ddevelocity.user.config=${{ github.workspace }}/.mvn/develocity-asf.xml
4646
MAVEN_OPTS: -Daether.connector.http.connectionMaxTtl=25 -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.intervalMax=30000
47+
MVNW_VERBOSE: true
4748
OZONE_WITH_COVERAGE: ${{ github.event_name != 'pull_request' }}
4849

4950
permissions: { }

‎.github/workflows/populate-cache.yml‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ on:
2929
- '.github/actions/setup-java-with-cache/action.yml'
3030
- '.github/workflows/populate-cache.yml'
3131
- '.mvn/extensions.xml'
32+
- '.mvn/wrapper/maven-wrapper.properties'
3233
workflow_call:
3334
workflow_dispatch:
3435

@@ -44,6 +45,7 @@ env:
4445
TEST_JAVA_VERSION: 25 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
4546
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml -Ddevelocity.user.config=${{ github.workspace }}/.mvn/develocity-asf.xml
4647
MAVEN_OPTS: -Daether.connector.http.connectionMaxTtl=25 -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.intervalMax=30000
48+
MVNW_VERBOSE: true
4749
SKIP_APACHE_VERIFY_DOWNLOAD: false
4850

4951
jobs:
@@ -62,14 +64,19 @@ jobs:
6264
java-version: ${{ env.TEST_JAVA_VERSION }}
6365
only-exact-hit: true
6466

67+
- name: Fetch plugin dependencies
68+
if: steps.restore-cache.outputs.cache-hit != 'true'
69+
run: ./mvnw $MAVEN_ARGS --no-transfer-progress --show-version dependency:resolve-plugins
70+
6571
- name: Get versions
6672
id: versions
6773
run: |
68-
NODEJS_VERSION=$(mvn help:evaluate -N -Dexpression=nodejs.version -q -DforceStdout -Dscan=false)
74+
source hadoop-ozone/dev-support/checks/_lib.sh
75+
NODEJS_VERSION=$(get_version nodejs)
6976
echo "NODEJS_VERSION=$NODEJS_VERSION" >> $GITHUB_ENV
70-
HADOOP_VERSION=$(mvn help:evaluate -N -Dexpression=hadoop.version -q -DforceStdout -Dscan=false)
77+
HADOOP_VERSION=$(get_version hadoop)
7178
echo "HADOOP_VERSION=$HADOOP_VERSION" >> $GITHUB_ENV
72-
RANGER_VERSION=$(mvn help:evaluate -N -Dexpression=ranger.version -q -DforceStdout -Dscan=false)
79+
RANGER_VERSION=$(get_version ranger)
7380
echo "RANGER_VERSION=$RANGER_VERSION" >> $GITHUB_ENV
7481
7582
- name: Restore Hadoop
@@ -133,21 +140,17 @@ jobs:
133140
if: steps.restore-cache.outputs.cache-hit != 'true' && steps.restore-nodejs.outputs.cache-hit != 'true'
134141
run: dev-support/ci/download-nodejs.sh
135142

136-
- name: Fetch plugin dependencies
137-
if: steps.restore-cache.outputs.cache-hit != 'true'
138-
run: mvn $MAVEN_ARGS --no-transfer-progress --show-version dependency:resolve-plugins
139-
140143
- name: Fetch dependencies
141144
if: steps.restore-cache.outputs.cache-hit != 'true'
142-
run: mvn $BUILD_ARGS $MAVEN_ARGS --no-transfer-progress --show-version -Pgo-offline clean verify
145+
run: ./mvnw $BUILD_ARGS $MAVEN_ARGS --no-transfer-progress --show-version -Pgo-offline clean verify
143146

144147
- name: Delete Ozone jars from repo
145148
if: steps.restore-cache.outputs.cache-hit != 'true'
146149
run: rm -fr ~/.m2/repository/org/apache/ozone
147150

148151
- name: List repo contents
149152
if: always()
150-
run: find ~/.m2/repository -type f | sort | xargs ls -lh
153+
run: find ~/.m2 -type f | sort | xargs ls -lh
151154

152155
- name: Save cache for Maven dependencies
153156
if: steps.restore-cache.outputs.cache-hit != 'true'
@@ -156,4 +159,5 @@ jobs:
156159
path: |
157160
~/.m2/repository/*/*/*
158161
!~/.m2/repository/org/apache/ozone
162+
~/.m2/wrapper
159163
key: ${{ steps.restore-cache.outputs.cache-key }}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
wrapperVersion=3.3.4
2+
distributionType=only-script
3+
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.10.1/apache-maven-3.10.1-bin.zip

‎dev-support/ci/download-nodejs.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ mkdir -p "${REPO_DIR}"
3434
declare -i rc=1
3535

3636
if [[ -z "${NODEJS_VERSION:-}" ]]; then
37-
NODEJS_VERSION=$(mvn help:evaluate -Dexpression=nodejs.version -q -DforceStdout -Dscan=false)
37+
source hadoop-ozone/dev-support/checks/_lib.sh
38+
NODEJS_VERSION=$(get_version nodejs)
3839
fi
3940

4041
if [[ -n "${NODEJS_VERSION}" ]]; then

‎dev-support/ci/errorprone.bats‎

Lines changed: 42 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -15,88 +15,68 @@
1515
# limitations under the License.
1616

1717
setup() {
18-
TEST_TMPDIR=$(mktemp -d)
19-
export TEST_TMPDIR
20-
export OUTPUT_DIR="${TEST_TMPDIR}/report"
21-
mkdir -p "${TEST_TMPDIR}/bin" "${OUTPUT_DIR}"
22-
echo 9 > "${OUTPUT_DIR}/failures"
23-
export MAVEN_EXIT_CODE=1
24-
cat > "${TEST_TMPDIR}/bin/mvn" <<'EOF'
25-
#!/usr/bin/env bash
26-
rm -rf "${OUTPUT_DIR}"
27-
printf '%s\n' "$*" > "${TEST_TMPDIR}/mvn-args"
28-
printf '%s\n' \
29-
"[WARNING] The following options were not recognized by any processor: '[artifactId]'" \
30-
"[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding" \
31-
"[WARNING] /src/Custom.java:[3,1] [Custom-Rule_1] custom finding" \
32-
"[ERROR] /src/Format.java:[2,1] [FormatString] error finding" \
33-
"[ERROR] /src/Format.java:[2,1] [FormatString] error finding" \
34-
"/src/Raw.java:4: warning: [RawWarning] raw warning" \
35-
"[ERROR] Failed to execute goal example:example: failed"
36-
printf '%s\n' "/src/Raw.java:5: error: [RawError] raw error" >&2
37-
exit "${MAVEN_EXIT_CODE}"
38-
EOF
39-
chmod +x "${TEST_TMPDIR}/bin/mvn"
40-
export PATH="${TEST_TMPDIR}/bin:${PATH}"
18+
REPORT_DIR=$(mktemp -d)
19+
export REPORT_DIR
4120
}
4221

4322
teardown() {
44-
rm -rf "${TEST_TMPDIR}"
23+
rm -rf "${REPORT_DIR}"
4524
}
4625

4726
@test "Error Prone reports Maven and javac findings and fails for errors" {
48-
run hadoop-ozone/dev-support/checks/errorprone.sh
27+
export rc=1
28+
cat > "${REPORT_DIR}/output.log" <<'EOF'
29+
[WARNING] The following options were not recognized by any processor: '[artifactId]'
30+
[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding
31+
[WARNING] /src/Custom.java:[3,1] [Custom-Rule_1] custom finding
32+
[ERROR] /src/Format.java:[2,1] [FormatString] error finding
33+
[ERROR] /src/Format.java:[2,1] [FormatString] error finding
34+
/src/Raw.java:4: warning: [RawWarning] raw warning
35+
/src/Raw.java:5: error: [RawError] raw error
36+
[ERROR] Failed to execute goal example:example: failed
37+
EOF
38+
39+
run hadoop-ozone/dev-support/checks/_errorprone_report.sh
4940

5041
[ "$status" -eq 1 ]
51-
grep -q -- '-DskipTests' "${TEST_TMPDIR}/mvn-args"
52-
grep -q -- '-DskipShade' "${TEST_TMPDIR}/mvn-args"
53-
grep -q -- 'clean package' "${TEST_TMPDIR}/mvn-args"
54-
[ "$(wc -l < "${OUTPUT_DIR}/diagnostics.txt")" -eq 5 ]
55-
grep -q '^\[WARNING\].*\[JdkObsolete\]' "${OUTPUT_DIR}/diagnostics.txt"
56-
grep -q '^\[ERROR\].*\[FormatString\]' "${OUTPUT_DIR}/diagnostics.txt"
57-
grep -q '^\[WARNING\].*\[Custom-Rule_1\]' "${OUTPUT_DIR}/diagnostics.txt"
58-
grep -q '^/src/Raw.java:4: warning: \[RawWarning\]' "${OUTPUT_DIR}/diagnostics.txt"
59-
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${OUTPUT_DIR}/diagnostics.txt"
60-
! grep -q 'Failed to execute goal' "${OUTPUT_DIR}/diagnostics.txt"
61-
[ "$(wc -l < "${OUTPUT_DIR}/summary.txt")" -eq 2 ]
62-
grep -q '^\[ERROR\].*\[FormatString\]' "${OUTPUT_DIR}/summary.txt"
63-
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${OUTPUT_DIR}/summary.txt"
64-
[ "$(< "${OUTPUT_DIR}/failures")" -eq 2 ]
42+
[ "$(wc -l < "${REPORT_DIR}/diagnostics.txt")" -eq 5 ]
43+
grep -q '^\[WARNING\].*\[JdkObsolete\]' "${REPORT_DIR}/diagnostics.txt"
44+
grep -q '^\[ERROR\].*\[FormatString\]' "${REPORT_DIR}/diagnostics.txt"
45+
grep -q '^\[WARNING\].*\[Custom-Rule_1\]' "${REPORT_DIR}/diagnostics.txt"
46+
grep -q '^/src/Raw.java:4: warning: \[RawWarning\]' "${REPORT_DIR}/diagnostics.txt"
47+
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${REPORT_DIR}/diagnostics.txt"
48+
! grep -q 'Failed to execute goal' "${REPORT_DIR}/diagnostics.txt"
49+
[ "$(wc -l < "${REPORT_DIR}/summary.txt")" -eq 2 ]
50+
grep -q '^\[ERROR\].*\[FormatString\]' "${REPORT_DIR}/summary.txt"
51+
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${REPORT_DIR}/summary.txt"
52+
[ "$(< "${REPORT_DIR}/failures")" -eq 2 ]
6553
}
6654

6755
@test "Error Prone warnings do not fail the check" {
68-
export MAVEN_EXIT_CODE=0
69-
cat > "${TEST_TMPDIR}/bin/mvn" <<'EOF'
70-
#!/usr/bin/env bash
71-
rm -rf "${OUTPUT_DIR}"
72-
printf '%s\n' \
73-
"[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding" \
74-
"/src/Raw.java:4: warning: [RawWarning] raw warning"
75-
exit "${MAVEN_EXIT_CODE}"
56+
export rc=0
57+
cat > "${REPORT_DIR}/output.log" <<'EOF'
58+
[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding
59+
/src/Raw.java:4: warning: [RawWarning] raw warning
7660
EOF
77-
chmod +x "${TEST_TMPDIR}/bin/mvn"
7861

79-
run hadoop-ozone/dev-support/checks/errorprone.sh
62+
run hadoop-ozone/dev-support/checks/_errorprone_report.sh
8063

8164
[ "$status" -eq 0 ]
82-
[ "$(wc -l < "${OUTPUT_DIR}/diagnostics.txt")" -eq 2 ]
83-
[ "$(< "${OUTPUT_DIR}/failures")" -eq 0 ]
84-
[ ! -s "${OUTPUT_DIR}/summary.txt" ]
65+
[ "$(wc -l < "${REPORT_DIR}/diagnostics.txt")" -eq 2 ]
66+
[ "$(< "${REPORT_DIR}/failures")" -eq 0 ]
67+
[ ! -s "${REPORT_DIR}/summary.txt" ]
8568
}
8669

8770
@test "Error Prone reports an unknown Maven failure" {
88-
cat > "${TEST_TMPDIR}/bin/mvn" <<'EOF'
89-
#!/usr/bin/env bash
90-
rm -rf "${OUTPUT_DIR}"
91-
echo "[ERROR] Failed to execute goal example:example: failed"
92-
exit "${MAVEN_EXIT_CODE}"
71+
export rc=1
72+
cat > "${REPORT_DIR}/output.log" <<'EOF'
73+
[ERROR] Failed to execute goal example:example: failed
9374
EOF
94-
chmod +x "${TEST_TMPDIR}/bin/mvn"
9575

96-
run hadoop-ozone/dev-support/checks/errorprone.sh
76+
run hadoop-ozone/dev-support/checks/_errorprone_report.sh
9777

9878
[ "$status" -eq 1 ]
99-
[ ! -s "${OUTPUT_DIR}/diagnostics.txt" ]
100-
[ "$(< "${OUTPUT_DIR}/summary.txt")" = "Unknown failure, check output.log" ]
101-
[ "$(< "${OUTPUT_DIR}/failures")" -eq 1 ]
79+
[ ! -s "${REPORT_DIR}/diagnostics.txt" ]
80+
[ "$(< "${REPORT_DIR}/summary.txt")" = "Unknown failure, check output.log" ]
81+
[ "$(< "${REPORT_DIR}/failures")" -eq 1 ]
10282
}

‎hadoop-ozone/dev-support/checks/_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ else
3737
fi
3838

3939
export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
40-
mvn ${MAVEN_OPTIONS} clean "$@" | tee output.log
40+
./mvnw ${MAVEN_OPTIONS} clean "$@" | tee output.log
4141
rc=$?
4242

4343
mkdir -p "$REPORT_DIR" # after `mvn clean`
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
set -u -o pipefail
19+
20+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
21+
22+
DIAGNOSTIC_FILE="$REPORT_DIR/diagnostics.txt"
23+
REPORT_FILE="$REPORT_DIR/summary.txt"
24+
25+
MAVEN_DIAGNOSTIC_PATTERN='^\[(ERROR|WARNING)\] .*:\[[0-9]+,[0-9]+\] \[[^]]+\]'
26+
JAVAC_DIAGNOSTIC_PATTERN='^.*:[0-9]+: (error|warning): \[[^]]+\]'
27+
MAVEN_ERROR_PATTERN='^\[ERROR\] .*:\[[0-9]+,[0-9]+\] \[[^]]+\]'
28+
JAVAC_ERROR_PATTERN='^.*:[0-9]+: error: \[[^]]+\]'
29+
ERROR_DIAGNOSTIC_PATTERN="${MAVEN_ERROR_PATTERN}|${JAVAC_ERROR_PATTERN}"
30+
31+
grep -E "${MAVEN_DIAGNOSTIC_PATTERN}|${JAVAC_DIAGNOSTIC_PATTERN}" "${REPORT_DIR}/output.log" \
32+
| awk '!seen[$0]++' > "$DIAGNOSTIC_FILE"
33+
34+
grep -E "$ERROR_DIAGNOSTIC_PATTERN" "$DIAGNOSTIC_FILE" > "$REPORT_FILE" || true
35+
36+
source "${DIR}/_post_process.sh"

‎hadoop-ozone/dev-support/checks/_lib.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,9 @@ create_aws_dir() {
9595
popd
9696
fi
9797
}
98+
99+
get_version() {
100+
local dep="$1"
101+
shift
102+
MVNW_VERBOSE=false ./mvnw --non-recursive --quiet help:evaluate -Dexpression="${dep}.version" -DforceStdout -Dscan=false "$@"
103+
}

0 commit comments

Comments
 (0)