datafusion-distributed/.github/workflows/ci.yml at main · datafusion-contrib/datafusion-distributed · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: CI

on:
  pull_request:
  push:
    branches:
      - main

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo build

  clippy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
        with:
          components: clippy
      - run: cargo clippy -p datafusion-distributed -p datafusion-distributed-iceberg --all-targets --all-features -- -D warnings

  unit-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true
      - uses: ./.github/actions/setup
      - run: cargo test --features integration

  tpch-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpch --test tpch_correctness_test
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  tpch-sorted-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpch --test tpch_sorted_correctness_test
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  tpch-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpch --test tpch_plans_test

  tpch-sorted-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpch --test tpch_sorted_plans_test

  tpcds-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        shard: [ "01", "02", "03", "04", "05", "06", "07", "08", "09", "10" ]
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }}
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  tpcds-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpcds --test tpcds_plans_test

  clickbench-correctness-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - uses: actions/cache@v4
        with:
          path: testdata/clickbench_parquet/
          key: "clickbench_parquet"
      - run: cargo test --features clickbench --test clickbench_correctness_test
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  clickbench-plans-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - uses: actions/cache@v4
        with:
          path: testdata/clickbench_parquet/
          key: "clickbench_parquet"
      - run: cargo test --features clickbench --test clickbench_plans_test

  iceberg-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools

  chaos-test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        planning_mode: [ "adaptive", "static" ]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
      - run: cargo test --features tpch,chaos-tests --test chaos
        env:
          ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }}

  format-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup
        with:
          components: rustfmt
      - run: cargo fmt --all -- --check

  typos:
    name: Spell Check with Typos
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      # Version fixed on purpose. typos uses heuristics and upgrading it may cause
      # new failures. Upgrade manually when needed.
      - name: Install typos
        uses: taiki-e/install-action@v2
        with:
          tool: typos@1.46.1
      - name: Run typos check
        run: .github/ci/scripts/typos_check.sh