diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000000..6aa3be3b11 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,44 @@ +name: Set up Gobierto +description: Checks out dependencies and prepares the toolchain shared by every CI job + +inputs: + engine-deploy-key: + description: SSH private key with read access to the private custom engine repositories + required: true + +runs: + using: composite + steps: + # script/custom_engines_ci_setup and the symlink scripts it calls in the + # engine repositories address this checkout as $DEV_DIR/gobierto. Those + # scripts print a hint and exit 0 when DEV_DIR is unset, so a missing value + # leaves the engines uninstalled without failing the step. + - name: Expose the checkout at $DEV_DIR/gobierto + shell: bash + run: | + ln -sfn "$GITHUB_WORKSPACE" "$HOME/gobierto" + echo "DEV_DIR=$HOME" >> "$GITHUB_ENV" + + - name: Trust the github.com host key + shell: bash + run: | + mkdir -p ~/.ssh + ssh-keyscan -t rsa,ecdsa,ed25519 github.com >> ~/.ssh/known_hosts + + - uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ inputs.engine-deploy-key }} + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + bundler-cache: true + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + + - name: Install JavaScript dependencies + shell: bash + run: yarn install --frozen-lockfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..636472ccf9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,339 @@ +name: CI + +on: + push: + branches-ignore: + - staging + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +env: + RAILS_ENV: test + NODE_ENV: test + NODE_OPTIONS: --max-old-space-size=6144 --openssl-legacy-provider + RUBY_YJIT_ENABLE: 1 + # Disable spring so bin/rails works. See: https://github.com/rails/spring/pull/546 + DISABLE_SPRING: true + BUNDLE_WITHOUT: development + # Retries flaky tests, see Minitest::Retry in test/test_helper.rb + RETRY_FAILING_TEST: 1 + ELASTICSEARCH_URL: http://localhost:9200 + ELASTICSEARCH_WRITING_URL: http://localhost:9200 + CUSTOM_ENGINE_NAME_1: ${{ vars.CUSTOM_ENGINE_NAME_1 }} + PGHOST: localhost + PGUSER: gobierto + PGPASSWORD: gobierto + PSQL_PAGER: '' + PG_HOST: localhost + PG_USERNAME: gobierto + PG_PASSWORD: gobierto + +jobs: + build: + name: build + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: gobierto + POSTGRES_DB: gobierto_test + POSTGRES_PASSWORD: gobierto + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U gobierto" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + elasticsearch: + image: elasticsearch:7.17.19 + env: + discovery.type: single-node + xpack.security.enabled: 'false' + ES_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -fsS http://localhost:9200/_cluster/health" + --health-interval 10s + --health-timeout 5s + --health-retries 20 + --ulimit memlock=-1:-1 + redis: + image: redis:7.0.2 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup + with: + engine-deploy-key: ${{ secrets.CUSTOM_ENGINE_DEPLOY_KEY }} + + - name: Copy database config + run: cp config/database.yml.example config/database.yml + + - name: Install custom engines + run: script/custom_engines_ci_setup + + - name: Compile I18n JS file + run: bin/rails i18n:js:export + + - name: Compile assets + run: bin/rails assets:precompile + + # The compiled assets travel to the test jobs through an artifact, not + # through a cache: a cache key is immutable, so a branch that keeps pushing + # JS changes would have its test jobs served the bundle of its first run. + - uses: actions/upload-artifact@v4 + with: + name: build-output + path: | + db + public/assets + retention-days: 1 + + test: + name: ${{ matrix.name }}${{ matrix.shards && format(' {0}/{1}', matrix.shard, matrix.shards) || '' }} + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: admin + shard: 1 + shards: 2 + files: find test -path '*gobierto_admin*' -name '*_test.rb' | grep -v 'gobierto_admin/gobierto_' + - name: admin + shard: 2 + shards: 2 + files: find test -path '*gobierto_admin*' -name '*_test.rb' | grep -v 'gobierto_admin/gobierto_' + - name: attachments + files: find test -path '*gobierto_attachments*' -name '*_test.rb' + - name: budgets + files: find test -path '*gobierto_budgets*' -name '*_test.rb' + - name: calendars + files: find test -path '*gobierto_calendars*' -name '*_test.rb' + - name: cms + files: find test -path '*gobierto_cms*' -name '*_test.rb' + - name: common + files: find test -path '*gobierto_common*' -name '*_test.rb' + - name: core + files: find test -path '*gobierto_core*' -name '*_test.rb' + - name: dashboards + files: find test -path '*gobierto_dashboards*' -name '*_test.rb' + - name: data + files: find test -path '*gobierto_data*' -name '*_test.rb' + - name: exports + files: find test -path '*gobierto_exports*' -name '*_test.rb' + - name: indicators + files: find test -path '*gobierto_indicators*' -name '*_test.rb' + - name: investments + files: find test -path '*gobierto_investments*' -name '*_test.rb' + - name: observatory + files: find test -path '*gobierto_observatory*' -name '*_test.rb' + - name: people + shard: 1 + shards: 2 + files: find test -path '*gobierto_people*' -name '*_test.rb' + - name: people + shard: 2 + shards: 2 + files: find test -path '*gobierto_people*' -name '*_test.rb' + - name: plans + shard: 1 + shards: 2 + files: find test -path '*gobierto_plans*' -name '*_test.rb' + - name: plans + shard: 2 + shards: 2 + files: find test -path '*gobierto_plans*' -name '*_test.rb' + - name: visualizations + files: find test -path '*gobierto_visualizations*' -name '*_test.rb' + - name: others + files: find test -not -path '*gobierto_*' -name '*_test.rb' + - name: engines + files: find -L vendor/gobierto_engines/ -name '*_test.rb' + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: gobierto + POSTGRES_DB: gobierto_test + POSTGRES_PASSWORD: gobierto + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U gobierto" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + elasticsearch: + image: elasticsearch:7.17.19 + env: + discovery.type: single-node + xpack.security.enabled: 'false' + ES_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -fsS http://localhost:9200/_cluster/health" + --health-interval 10s + --health-timeout 5s + --health-retries 20 + --ulimit memlock=-1:-1 + redis: + image: redis:7.0.2 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup + with: + engine-deploy-key: ${{ secrets.CUSTOM_ENGINE_DEPLOY_KEY }} + + - uses: actions/download-artifact@v4 + with: + name: build-output + + # Timings recorded on master balance the shards. Without them the split + # falls back to file size, which self-corrects after one master build. + - uses: actions/cache/restore@v4 + with: + path: test/timings.json + key: test-timings-${{ github.sha }} + restore-keys: test-timings- + + # Rails dumps db/structure.sql with pg_dump, which must match the server + - name: Install PostgreSQL 15 client + run: | + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/pgdg.gpg + echo "deb [signed-by=/usr/share/keyrings/pgdg.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + sudo apt-get -y install postgresql-client-15 + + - name: Wait for services + run: | + timeout 60 bash -c 'until pg_isready -q; do sleep 1; done' + timeout 180 bash -c "until curl -fsS $ELASTICSEARCH_URL/_cluster/health > /dev/null; do sleep 2; done" + timeout 60 bash -c 'until (exec 3<>/dev/tcp/localhost/6379); do sleep 1; done' + + - name: Copy database config + run: cp config/database.yml.example config/database.yml + + - name: Setup the database + run: bin/rails db:create db:migrate + + - name: Install custom engines + run: script/custom_engines_ci_setup + + - name: Setup budgets seeds + run: | + bin/rails gobierto_budgets_data:elastic_search_schemas:reset + bin/rails gobierto_budgets_data:elastic_search_schemas:create + bin/rails gobierto_budgets_data:data:reset + bin/rails gobierto_budgets_data:data:create + bin/rails gobierto_budgets:fixtures:load + echo '::BudgetsSeeder.seed!' | bin/rails c + + - name: Run tests + env: + TEST_FILES_CMD: ${{ matrix.files }} + run: | + eval "$TEST_FILES_CMD" \ + | script/ci/split_tests "${{ matrix.shard || 1 }}" "${{ matrix.shards || 1 }}" \ + | xargs --no-run-if-empty bin/rails test + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: test-reports-${{ matrix.name }}-${{ matrix.shard || 1 }} + path: test/reports + if-no-files-found: ignore + retention-days: 7 + + collect_timings: + name: collect timings + needs: test + # Runs on partial failures too: the shards that did finish still carry + # usable timings. Skipped runs would only produce an empty file. + if: ${{ !cancelled() && needs.test.result != 'skipped' && github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + + - uses: actions/download-artifact@v4 + with: + pattern: test-reports-* + path: reports + + - name: Record how long every test file took + id: timings + run: | + script/ci/collect_timings reports > test/timings.json + count=$(ruby -rjson -e 'puts JSON.parse(File.read("test/timings.json")).size') + echo "count=$count" >> "$GITHUB_OUTPUT" + echo "Recorded timings for $count test files" + + # An empty file would become the newest cache entry and shadow good data + - uses: actions/cache/save@v4 + if: steps.timings.outputs.count != '0' + with: + path: test/timings.json + key: test-timings-${{ github.sha }} + + deploy: + name: production deploy + needs: [build, test] + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/checkout@v4 + + - name: Deploy master branch + env: + DEPLOY_BOT_TOKEN: ${{ secrets.DEPLOY_BOT_TOKEN }} + GOBIERTO_PRODUCTION_DEPLOY_URL: ${{ secrets.GOBIERTO_PRODUCTION_DEPLOY_URL }} + run: script/production_deploy.sh + + # Not part of the automatic pipeline, matching the CircleCI setup where this + # job was defined but never listed in the workflow. + test_javascript: + name: javascript + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + + - run: yarn install --frozen-lockfile + + - run: yarn run test diff --git a/.gitignore b/.gitignore index 410b684c06..71d05090bc 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ config/deploy/production.rb /coverage test/reports +# Per-file test durations, rebuilt by CI to balance test shards +test/timings.json + # Uploaded files /public/system/attachments /public/packs diff --git a/script/ci/collect_timings b/script/ci/collect_timings new file mode 100755 index 0000000000..e9adcf4dde --- /dev/null +++ b/script/ci/collect_timings @@ -0,0 +1,34 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Sums the runtime of every test case per file from the JUnit XML that +# minitest-ci writes to test/reports, and prints it as JSON on stdout. +# +# REPORT_DIR is searched recursively, so it also accepts the directory a CI job +# downloaded several runs of reports into. +# +# The result feeds script/ci/split_tests on later runs so shards stay balanced. +# +# Usage: script/ci/collect_timings [REPORT_DIR] + +require "json" +require "rexml/document" + +report_dir = ARGV[0] || File.join("test", "reports") + +timings = Hash.new(0.0) + +Dir.glob(File.join(report_dir, "**", "*.xml")).sort.each do |path| + document = REXML::Document.new(File.read(path)) + + REXML::XPath.each(document, "//testcase") do |testcase| + file = testcase.attributes["file"] + next if file.nil? || file.empty? + + timings[file] += testcase.attributes["time"].to_f + end +rescue REXML::ParseException => e + warn "Skipping unparseable report #{path}: #{e.message}" +end + +puts JSON.pretty_generate(timings.sort.to_h) diff --git a/script/ci/split_tests b/script/ci/split_tests new file mode 100755 index 0000000000..459f26db61 --- /dev/null +++ b/script/ci/split_tests @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Splits a list of test files (read from stdin) into balanced shards. +# +# Weights come from test/timings.json, a map of file path to seconds produced by +# script/ci/collect_timings on a previous run. Files without a recorded time get +# the median of the known ones; when no timings exist at all, file size is used +# as a rough proxy. +# +# Usage: | script/ci/split_tests SHARD SHARDS + +require "json" + +shard = Integer(ARGV[0] || 1) +shards = Integer(ARGV[1] || 1) + +abort "SHARD must be between 1 and #{shards}" unless (1..shards).cover?(shard) + +files = $stdin.read.split("\n").map(&:strip).reject(&:empty?).uniq.sort + +if shards == 1 + puts files + exit +end + +timings_path = File.join(Dir.pwd, "test", "timings.json") +timings = File.exist?(timings_path) ? JSON.parse(File.read(timings_path)) : {} + +known = files.filter_map { |file| timings[file] } +fallback = if known.empty? + nil + else + sorted = known.sort + sorted[sorted.size / 2] + end + +weight_of = lambda do |file| + timings[file] || fallback || (File.size?(file) || 0).to_f +end + +# Longest processing time first: heaviest files go to the emptiest shard. +loads = Array.new(shards, 0.0) +buckets = Array.new(shards) { [] } + +files.sort_by { |file| [-weight_of.call(file), file] }.each do |file| + index = (0...shards).min_by { |i| [loads[i], i] } + loads[index] += weight_of.call(file) + buckets[index] << file +end + +puts buckets[shard - 1].sort