Repository navigation
Reuse immutable cache for bounded SlateDB reads #5225
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| artifacts_only: | |
| description: Build tested browser SDK and server artifacts without full release validation | |
| type: boolean | |
| default: false | |
| source_revision: | |
| description: Exact 40-character commit SHA at the dispatched ref (required for artifacts_only) | |
| type: string | |
| required: false | |
| pull_request: | |
| types: [opened, reopened, synchronize, ready_for_review] | |
| push: | |
| branches: | |
| - main | |
| - master | |
| concurrency: | |
| # A rerun retains its original event payload. Draft reruns must never cancel | |
| # ready-candidate validation, even when they reference the same head commit. | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.draft && 'draft' || 'ready' }}-${{ inputs.artifacts_only && 'artifacts' || 'validation' }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| defaults: | |
| run: | |
| # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun | |
| # https://linux.die.net/man/1/bash | |
| shell: bash --noprofile --norc -o errexit -o nounset -o pipefail -o xtrace -O nullglob -O dotglob {0} | |
| jobs: | |
| release-ready: | |
| # Draft reruns must not overwrite the required ready-candidate check. | |
| name: ${{ inputs.artifacts_only && 'Artifacts only - not release validation' || github.event.pull_request.draft && 'Draft - full CI deferred' || 'Release ready' }} | |
| if: always() && inputs.artifacts_only != true && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) | |
| needs: [merge-reuse, promote-browser-sdk, promote-server-image, content-browser-sdk, changelog, cargo-config, cargo, js-sdk-test, preview-artifact-changes, preview-server-image] | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Require successful validation of the current candidate | |
| uses: actions/github-script@v7 | |
| env: | |
| VALIDATION_NEEDS: ${{ toJSON(needs) }} | |
| with: | |
| script: | | |
| const { assertReleaseReady } = await import(`${process.env.GITHUB_WORKSPACE}/scripts/release-candidate.mjs`); | |
| const currentPr = context.eventName === 'pull_request' | |
| ? (await github.rest.pulls.get({ ...context.repo, pull_number: context.issue.number })).data | |
| : undefined; | |
| assertReleaseReady({ | |
| needs: JSON.parse(process.env.VALIDATION_NEEDS), | |
| eventName: context.eventName, event: context.payload, currentPr, | |
| }); | |
| merge-reuse: | |
| name: Reuse tested merge | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| actions: read | |
| pull-requests: read | |
| outputs: | |
| reuse: ${{ steps.reuse.outputs.reuse }} | |
| run_id: ${{ steps.reuse.outputs.run_id }} | |
| revision: ${{ steps.reuse.outputs.revision }} | |
| server_artifact: ${{ steps.reuse.outputs.server_artifact }} | |
| content_only: ${{ steps.content.outputs.content_only }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Select content-only scope | |
| id: content | |
| run: node scripts/ci-content-scope.mjs | |
| - name: Validate explicit artifact source | |
| uses: actions/github-script@v7 | |
| env: | |
| ARTIFACTS_ONLY: ${{ inputs.artifacts_only || false }} | |
| SOURCE_REVISION: ${{ inputs.source_revision }} | |
| with: | |
| script: | | |
| const { assertArtifactRequest } = await import(`${process.env.GITHUB_WORKSPACE}/scripts/ci-artifact-request.mjs`); | |
| assertArtifactRequest({ | |
| artifactsOnly: process.env.ARTIFACTS_ONLY === 'true', | |
| eventName: context.eventName, | |
| requestedRevision: process.env.SOURCE_REVISION, | |
| workflowRevision: context.sha, | |
| }); | |
| - name: Find successful PR validation of the exact merged tree | |
| id: reuse | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { selectMergeReuse } = await import(`${process.env.GITHUB_WORKSPACE}/scripts/ci-merge-reuse.mjs`); | |
| await selectMergeReuse({ github, context, core }); | |
| promote-browser-sdk: | |
| name: Reuse tested browser SDK | |
| needs: merge-reuse | |
| if: needs.merge-reuse.outputs.reuse == 'true' | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| timeout-minutes: 15 | |
| permissions: | |
| actions: read | |
| contents: read | |
| env: | |
| CARGO_INCREMENTAL: '0' | |
| CARGO_PROFILE_DEV_DEBUG: '0' | |
| CARGO_PROFILE_TEST_DEBUG: '0' | |
| CARGO_PROFILE_RELEASE_DEBUG: '0' | |
| CARGO_PROFILE_BENCH_DEBUG: '0' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| - name: Download and verify complete browser artifact | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| SOURCE_REVISION: ${{ needs.merge-reuse.outputs.revision }} | |
| SOURCE_RUN: ${{ needs.merge-reuse.outputs.run_id }} | |
| run: node scripts/release-browser-artifact.mjs download-merged | |
| - name: Validate browser binaries for main cache | |
| id: cache-payload | |
| env: | |
| SOURCE_REVISION: ${{ needs.merge-reuse.outputs.revision }} | |
| run: node scripts/release-browser-artifact.mjs prepare-merged-cache | |
| - name: Check main browser binary cache | |
| if: steps.cache-payload.outputs.key != '' | |
| id: cache-lookup | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .ci-sdk-cache/browser | |
| key: ${{ steps.cache-payload.outputs.key }} | |
| lookup-only: true | |
| - name: Seed main browser binary cache without recompilation | |
| if: steps.cache-payload.outputs.key != '' && steps.cache-lookup.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: .ci-sdk-cache/browser | |
| key: ${{ steps.cache-payload.outputs.key }} | |
| - name: Record identical-tree artifact provenance | |
| env: | |
| SOURCE_REVISION: ${{ needs.merge-reuse.outputs.revision }} | |
| SOURCE_RUN: ${{ needs.merge-reuse.outputs.run_id }} | |
| run: | | |
| node --input-type=module - <<'EOF' | |
| import { readFileSync, writeFileSync } from 'node:fs'; | |
| const path = 'ci-artifact/browser.json'; | |
| const manifest = JSON.parse(readFileSync(path, 'utf8')); | |
| if (manifest.schemaVersion !== 1 || manifest.kind !== 'lix-browser-sdk' || | |
| manifest.sourceRevision !== process.env.SOURCE_REVISION) { | |
| throw new Error('Unexpected source artifact provenance'); | |
| } | |
| manifest.reusedFromRevision = manifest.sourceRevision; | |
| manifest.reusedFromRun = process.env.SOURCE_RUN; | |
| manifest.sourceRevision = process.env.GITHUB_SHA; | |
| writeFileSync(path, JSON.stringify(manifest, null, 2)); | |
| EOF | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lix-browser-sdk-${{ github.sha }} | |
| path: | | |
| packages/js-sdk/dist | |
| packages/storage-opfs/dist | |
| ci-artifact/browser.json | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ci-browser-build | |
| path: ci-artifact/browser.json | |
| retention-days: 90 | |
| if-no-files-found: error | |
| promote-server-image: | |
| name: Reuse tested server image | |
| needs: merge-reuse | |
| if: needs.merge-reuse.outputs.reuse == 'true' && needs.merge-reuse.outputs.server_artifact == 'true' | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| timeout-minutes: 10 | |
| permissions: | |
| actions: read | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lix-server-image-linux-x64-${{ needs.merge-reuse.outputs.revision }} | |
| run-id: ${{ needs.merge-reuse.outputs.run_id }} | |
| github-token: ${{ github.token }} | |
| path: ci-artifact | |
| - name: Promote tested server layers to the landed revision | |
| env: | |
| SOURCE_REVISION: ${{ needs.merge-reuse.outputs.revision }} | |
| SOURCE_RUN: ${{ needs.merge-reuse.outputs.run_id }} | |
| run: node scripts/ci-promote-server-image.mjs | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lix-server-image-linux-x64-${{ github.sha }} | |
| path: | | |
| ci-artifact/lix-server-image.tar | |
| ci-artifact/server-linux-x64.json | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| changelog: | |
| name: Changelog | |
| needs: merge-reuse | |
| if: inputs.artifacts_only != true && needs.merge-reuse.outputs.reuse != 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| outputs: | |
| rust: ${{ needs.merge-reuse.outputs.content_only == 'true' && 'false' || steps.scope.outputs.rust }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| # Include both parents of the tested PR merge for scope detection. | |
| fetch-depth: 2 | |
| # Rust validates the checkout merge tree; SDK jobs validate the PR head. | |
| # This evidence is reusable only after the entire workflow succeeds. | |
| - name: Record tested source trees | |
| env: | |
| SOURCE_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} | |
| run: | | |
| node --input-type=module - <<'EOF' | |
| import { execFileSync } from 'node:child_process'; | |
| import { writeFileSync } from 'node:fs'; | |
| const tree = ref => execFileSync('git', ['rev-parse', `${ref}^{tree}`], { encoding: 'utf8' }).trim(); | |
| writeFileSync('tested-source.json', JSON.stringify({ | |
| schemaVersion: 1, | |
| sourceRevision: process.env.SOURCE_REVISION, | |
| sourceTree: tree(process.env.SOURCE_REVISION), | |
| testedTree: tree('HEAD'), | |
| })); | |
| EOF | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ci-tested-source | |
| path: tested-source.json | |
| retention-days: 90 | |
| - name: Select Rust CI scope | |
| id: scope | |
| run: node scripts/ci-rust-scope.mjs | |
| - name: Validate change fragments | |
| run: node scripts/validate-changes.mjs | |
| - name: Validate server protocol docs | |
| run: node scripts/validate-server-protocol-docs.mjs | |
| - name: Validate CI workflow invariants | |
| run: node --test scripts/compatibility.test.mjs scripts/native-build.test.mjs scripts/ci-workflow.test.mjs scripts/ci-merge-reuse.test.mjs scripts/ci-rust-scope.test.mjs scripts/ci-content-scope.test.mjs scripts/ci-content-artifact.test.mjs scripts/ci-sdk-cache.test.mjs scripts/ci-build-sdk.test.mjs scripts/ci-test-browser.test.mjs scripts/release-browser-artifact.test.mjs scripts/ci-server-image.test.mjs scripts/ci-promote-server-image.test.mjs scripts/release.test.mjs scripts/release-candidate.test.mjs scripts/publish-plugin.test.mjs | |
| - name: Test server artifact promotion with Docker | |
| run: LIX_TEST_DOCKER_PROMOTION=1 node --test scripts/ci-promote-server-image.test.mjs | |
| cargo-config: | |
| name: Cargo config (${{ matrix.name }}) | |
| needs: merge-reuse | |
| if: inputs.artifacts_only != true && needs.merge-reuse.outputs.reuse != 'true' && needs.merge-reuse.outputs.content_only != 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Linux x64 | |
| runner: ubicloud-standard-2-ubuntu-2404 | |
| - name: macOS arm64 | |
| runner: macos-15 | |
| - name: Windows x64 | |
| runner: windows-2025 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Validate Cargo configuration and publish surface | |
| run: | | |
| node scripts/validate-publish-surface.mjs | |
| node --test scripts/release.test.mjs | |
| cargo: | |
| # Large Rust builds need memory and parallel compilation to target <10 min. | |
| name: Cargo ${{ matrix.name }} | |
| needs: changelog | |
| if: inputs.artifacts_only != true && needs.changelog.outputs.rust != 'false' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 45 | |
| env: | |
| # The root Cargo config otherwise sends --manifest-path tooling builds | |
| # to root/target, while the cache and reports expect tooling/target. | |
| CARGO_TARGET_DIR: ${{ github.workspace }}/${{ matrix.workspace }}/target | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Clippy | |
| task: clippy | |
| workspace: . | |
| cache_workspaces: | | |
| . -> target | |
| tooling -> target | |
| runner: ubicloud-standard-30-ubuntu-2404 | |
| # Consumer checks compile different feature/toolchain combinations. | |
| # Run them alongside lint instead of extending its critical path. | |
| - name: Compatibility | |
| task: compatibility | |
| workspace: . | |
| runner: ubicloud-standard-30-ubuntu-2404 | |
| - name: Test | |
| task: test | |
| workspace: . | |
| junit: target/nextest/ci/junit.xml | |
| runner: ubicloud-standard-30-ubuntu-2404 | |
| - name: Tooling Test | |
| task: tooling | |
| workspace: tooling | |
| junit: tooling/target/nextest/ci/junit.xml | |
| runner: ubicloud-standard-30-ubuntu-2404 | |
| - name: E2E Test | |
| task: e2e | |
| workspace: tooling | |
| junit: tooling/target/nextest/ci-e2e/junit.xml | |
| runner: ubicloud-standard-30-ubuntu-2404 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Prepare Linux build runner | |
| uses: ./.github/actions/prepare-linux-runner | |
| - name: set envs | |
| run: | | |
| grep -v "^#" << "EOF" >> "$GITHUB_ENV" | |
| CARGO_TERM_COLOR=always | |
| CARGO_INCREMENTAL=0 | |
| # keep aligned with config.toml | |
| CARGO_PROFILE_DEV_DEBUG=0 | |
| CARGO_PROFILE_TEST_DEBUG=0 | |
| CARGO_PROFILE_RELEASE_DEBUG=0 | |
| CARGO_PROFILE_BENCH_DEBUG=0 | |
| CARGO_NET_RETRY=10 | |
| RUSTUP_MAX_RETRIES=10 | |
| GIT_AUTHOR_NAME=github-actions[bot] | |
| GIT_COMMITTER_NAME=github-actions[bot] | |
| GIT_AUTHOR_EMAIL=41898282+github-actions[bot]@users.noreply.github.com | |
| GIT_COMMITTER_EMAIL=41898282+github-actions[bot]@users.noreply.github.com | |
| DEBIAN_FRONTEND=noninteractive | |
| BINSTALL_NO_CONFIRM=true | |
| EOF | |
| - name: Install native build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y llvm-dev libclang-dev clang mold | |
| - name: Install cargo-nextest | |
| if: matrix.junit | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: nextest | |
| # Main seeds dependency caches; PRs only restore them. Clippy executes | |
| # both workspaces, so retain tooling dependencies as well as root ones. | |
| - name: Restore Cargo dependency cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cargo-${{ matrix.task }} | |
| key: explicit-targets-v1 | |
| workspaces: ${{ matrix.cache_workspaces || matrix.workspace }} | |
| cache-targets: true | |
| cache-workspace-crates: false | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Restore compiler cache | |
| uses: ./.github/actions/compiler-cache | |
| with: | |
| scope: cargo-${{ matrix.task }} | |
| - name: Lint Rust workspace | |
| if: matrix.task == 'clippy' | |
| run: | | |
| cargo clippy --profile test --workspace --all-targets --all-features -- -D warnings | |
| # Keep tooling artifacts in the second cached target directory. | |
| export CARGO_TARGET_DIR="$GITHUB_WORKSPACE/tooling/target" | |
| cargo clippy --locked --manifest-path tooling/Cargo.toml --profile test --workspace --exclude lix_e2e --all-targets --all-features -- -D warnings | |
| cargo clippy --locked --manifest-path tooling/Cargo.toml --profile test -p lix_e2e --all-targets --features sdk-tests,plugin-tests,server-protocol,storage-benches,slatedb,root-replay-trace,system-allocation-profiler -- -D warnings | |
| # The one supported configuration nothing else here builds: `cfg(test)` | |
| # with `storage-benches` OFF, which is exactly what a plain | |
| # `cargo nextest run -p lix` — the dev loop `packages/lix/AGENTS.md` | |
| # documents — compiles. Every other step has the feature on | |
| # (`--all-features`), and a `--no-default-features` check has no test | |
| # targets at all, so a | |
| # `#[cfg(test)]` item that calls the feature-only `storage_bench` module | |
| # breaks `cargo nextest run -p lix` while clippy, the full test matrix and | |
| # the `--all-features` build all stay green. Two such call sites had | |
| # accumulated in `gc.rs` before this step existed. `check` rather than | |
| # `test`: the matrix below already runs the tests, this only has to prove | |
| # the configuration compiles. | |
| - name: Check lix test targets with default features | |
| if: matrix.task == 'compatibility' | |
| run: cargo check -p lix --tests | |
| - name: Verify stable Cargo can embed the local Rust SDK | |
| if: matrix.task == 'compatibility' | |
| run: | | |
| rustup toolchain install 1.94.0 --profile minimal | |
| rustup run 1.94.0 cargo metadata --locked --format-version 1 --no-deps --manifest-path packages/lix/Cargo.toml | |
| downstream="$(mktemp -d)" | |
| rustup run 1.94.0 cargo init --bin --vcs none --name downstream_lix_sdk "$downstream" | |
| rustup run 1.94.0 cargo add --manifest-path "$downstream/Cargo.toml" --path "$PWD/packages/lix" --no-default-features lix | |
| # Run from the consumer crate. Invoking Cargo from the Lix checkout | |
| # would inherit this repository's nightly-only `.cargo/config.toml`, | |
| # which a real external path dependency does not inherit. | |
| # Pin the child compiler as well as Cargo. The rustc shim can | |
| # otherwise select the default nightly toolchain. | |
| (cd "$downstream" && RUSTUP_TOOLCHAIN=1.94.0 rustup run 1.94.0 cargo check) | |
| # `--all-features` is deliberate: it is the only thing that compiles | |
| # feature-gated sources such as `packages/lix/src/storage_bench.rs` | |
| # (behind `lix/storage-benches`), where a stale assertion once survived | |
| # roughly ten pull requests because a plain `cargo test -p lix` never | |
| # built it. | |
| # | |
| # `lix_e2e` is excluded from that blanket only because its `tpch` | |
| # feature pulls `duckdb` with the `bundled` C++ build. No test needs it — | |
| # DuckDB is used solely by `[[bench]] tpch`, and `cargo test` never runs | |
| # benches — so it was minutes of CI spent compiling a dependency nothing | |
| # here executes. The crate is still compiled and tested below with every | |
| # feature its tests actually require. | |
| # | |
| # That feature list is exhaustive, not a guess: every `[[test]]` target in | |
| # `packages/e2e/Cargo.toml` declares `required-features` within | |
| # {sdk-tests, storage-benches, slatedb, rocksdb}, and `storage-benches` | |
| # implies `rocksdb`. `sdk-tests` covers the 11 targets folded in from the | |
| # former `lix_tests` crate; it turns on `lix/default_wasm_runtime`, which | |
| # those tests need and which the benches in that crate deliberately do not | |
| # build with. `root-replay-trace` carries no test of its own but gates | |
| # instrumentation sources, so it is listed to keep them compiling — | |
| # dropping it would recreate exactly the `storage_bench.rs` blind spot | |
| # described above. | |
| - name: Run Rust workspace tests with nextest | |
| # Unoptimized async test poll chains exceed Rust's 2 MiB test-thread | |
| # stack in existing file and sync paths. Give test threads enough | |
| # space to exercise the behavior instead of aborting before assertions. | |
| # This does not change the stack of production worker threads. | |
| # | |
| # `fail-fast = false` is required, not cosmetic. #1329 broke two tests; | |
| # the second sat invisible behind the first for hours because CI could | |
| # only ever report one. | |
| if: matrix.task == 'test' | |
| env: | |
| RUST_MIN_STACK: "16777216" | |
| run: cargo nextest run --config-file .config/nextest.toml --profile ci --cargo-profile test --workspace --all-features --lib --tests --timings | |
| - name: Run Rust tooling tests with nextest | |
| if: matrix.task == 'tooling' | |
| run: cargo nextest run --config-file .config/nextest.toml --profile ci --locked --manifest-path tooling/Cargo.toml --cargo-profile test --workspace --exclude lix_e2e --all-features --lib --tests --timings | |
| # Cargo's default test target selection also compiles every example even | |
| # though nextest cannot execute their main functions. The e2e package has | |
| # 30 examples; explicitly selecting tests avoids rebuilding all 30 while | |
| # Clippy above still checks their compilation. This package has no library | |
| # target, so passing --lib would fail before any tests could run. | |
| - name: Run Rust E2E tests with nextest | |
| if: matrix.task == 'e2e' | |
| env: | |
| RUST_MIN_STACK: "16777216" | |
| run: cargo nextest run --config-file .config/nextest.toml --profile ci-e2e --locked --manifest-path tooling/Cargo.toml --cargo-profile test -p lix_e2e --features sdk-tests,plugin-tests,server-protocol,storage-benches,slatedb,root-replay-trace --tests --timings | |
| - name: Upload Rust test timing reports | |
| if: matrix.junit && !cancelled() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rust-nextest-junit-${{ matrix.task }} | |
| path: ${{ matrix.junit }} | |
| if-no-files-found: error | |
| - name: Upload Cargo build timings | |
| if: matrix.junit && !cancelled() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rust-cargo-timings-${{ matrix.task }} | |
| path: ${{ matrix.workspace }}/target/cargo-timings/*.html | |
| if-no-files-found: warn | |
| # Nextest deliberately does not execute rustdoc tests. Keep their Cargo | |
| # feature/package scopes aligned with the nextest runs above. | |
| - name: Run Rust doctests | |
| if: matrix.task == 'test' | |
| run: cargo test --doc --profile test --workspace --all-features --no-fail-fast | |
| - name: Run Rust tooling doctests | |
| if: matrix.task == 'tooling' | |
| run: cargo test --doc --locked --manifest-path tooling/Cargo.toml --profile test --workspace --exclude lix_e2e --all-features --no-fail-fast | |
| # The e2e test compilation above already builds the real workspace plugin | |
| # artifacts. This one external fixture covers the distinct risk: whether | |
| # the packaged `lix` crate contains and exposes everything a plugin author | |
| # needs. `--no-verify` avoids compiling the native engine a second time; | |
| # compiling the extracted package for WASIp2 is the relevant verification. | |
| - name: Verify the packaged Lix plugin contract | |
| if: matrix.task == 'test' | |
| run: | | |
| rustup target add wasm32-wasip2 | |
| cargo package --locked \ | |
| -p lix-schema \ | |
| -p lix \ | |
| --no-verify | |
| package_archive="$(find target/package -maxdepth 1 -type f -name 'lix-[0-9]*.crate' | head -n 1)" | |
| schema_archive="$(find target/package -maxdepth 1 -type f -name 'lix-schema-*.crate' | head -n 1)" | |
| tar -tf "$package_archive" | | |
| grep -E '^[^/]+/wit/lix-plugin\.wit$' | |
| packaged_lix="$(mktemp -d)" | |
| packaged_schema="$(mktemp -d)" | |
| tar -xf "$package_archive" -C "$packaged_lix" --strip-components=1 | |
| tar -xf "$schema_archive" -C "$packaged_schema" --strip-components=1 | |
| downstream="$(mktemp -d)" | |
| cargo init --lib --name downstream_lix_plugin "$downstream" | |
| printf '\n[patch.crates-io]\nlix-schema = { path = "%s" }\n' \ | |
| "$packaged_schema" \ | |
| >> "$downstream/Cargo.toml" | |
| cargo add --manifest-path "$downstream/Cargo.toml" --path "$packaged_lix" lix | |
| cp packages/lix/examples/plugin_minimal.rs "$downstream/src/lib.rs" | |
| cargo build --manifest-path "$downstream/Cargo.toml" --target wasm32-wasip2 | |
| - name: Report remaining runner resources | |
| if: always() | |
| run: | | |
| df -h "$GITHUB_WORKSPACE" | |
| free -h | |
| if command -v sccache >/dev/null; then sccache --show-stats; fi | |
| js-sdk-test: | |
| name: JS SDK ${{ matrix.runtime == 'browser' && 'Browser' || 'Native' }} Test | |
| needs: merge-reuse | |
| if: needs.merge-reuse.outputs.reuse != 'true' && needs.merge-reuse.outputs.content_only != 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) | |
| runs-on: ubicloud-standard-30-ubuntu-2404 | |
| timeout-minutes: 45 | |
| env: | |
| # Pull request workflows normally check out GitHub's synthetic merge | |
| # commit. Consumers pin the real source commit as a submodule, so build | |
| # reusable artifacts from that exact commit instead. | |
| LIX_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runtime: ${{ fromJSON(inputs.artifacts_only && '["browser"]' || '["native", "browser"]') }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.LIX_SOURCE_SHA }} | |
| - name: set envs | |
| run: | | |
| grep -v "^#" << "EOF" >> "$GITHUB_ENV" | |
| CARGO_TERM_COLOR=always | |
| CARGO_INCREMENTAL=0 | |
| # keep aligned with config.toml | |
| CARGO_PROFILE_DEV_DEBUG=0 | |
| CARGO_PROFILE_TEST_DEBUG=0 | |
| CARGO_PROFILE_RELEASE_DEBUG=0 | |
| CARGO_PROFILE_BENCH_DEBUG=0 | |
| CARGO_NET_RETRY=10 | |
| RUSTUP_MAX_RETRIES=10 | |
| LIX_NATIVE_PROFILE=test | |
| GIT_AUTHOR_NAME=github-actions[bot] | |
| GIT_COMMITTER_NAME=github-actions[bot] | |
| GIT_AUTHOR_EMAIL=41898282+github-actions[bot]@users.noreply.github.com | |
| GIT_COMMITTER_EMAIL=41898282+github-actions[bot]@users.noreply.github.com | |
| DEBIAN_FRONTEND=noninteractive | |
| BINSTALL_NO_CONFIRM=true | |
| EOF | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: npm | |
| cache-dependency-path: | | |
| packages/js-sdk/package-lock.json | |
| packages/storage-filesystem/package-lock.json | |
| packages/storage-opfs/package-lock.json | |
| - name: Fingerprint SDK binary build inputs | |
| id: binary-key | |
| run: node scripts/ci-sdk-cache.mjs key '${{ matrix.runtime }}' | |
| - name: Restore exact-input SDK binaries | |
| id: binaries | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .ci-sdk-cache/${{ matrix.runtime }} | |
| key: ${{ steps.binary-key.outputs.key }} | |
| - name: Validate restored SDK binaries | |
| id: binary-cache | |
| run: node scripts/ci-sdk-cache.mjs check '${{ matrix.runtime }}' '${{ steps.binary-key.outputs.key }}' | |
| - name: Prepare Linux build runner | |
| if: matrix.runtime == 'native' || steps.binary-cache.outputs.reuse != 'true' | |
| uses: ./.github/actions/prepare-linux-runner | |
| - name: Install LLVM build dependencies | |
| if: matrix.runtime == 'native' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y llvm-dev libclang-dev clang mold | |
| - name: Install mold for WebAssembly host tooling | |
| if: matrix.runtime == 'browser' && steps.binary-cache.outputs.reuse != 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y mold | |
| - name: Restore Rust cache | |
| if: matrix.runtime == 'native' || steps.binary-cache.outputs.reuse != 'true' | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| # Native uses test/dev profiles; Browser uses release Wasm. A single | |
| # immutable key lets the first writer seed only its own build modes. | |
| shared-key: ci-js-${{ matrix.runtime }} | |
| key: parallel-sdk-v1 | |
| cache-targets: false | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Restore compiler cache | |
| if: matrix.runtime == 'native' || steps.binary-cache.outputs.reuse != 'true' | |
| uses: ./.github/actions/compiler-cache | |
| with: | |
| scope: sdk-${{ matrix.runtime }} | |
| - name: Install plugin build target | |
| if: steps.binary-cache.outputs.reuse != 'true' | |
| run: rustup target add wasm32-wasip2 | |
| - name: Install WebAssembly build target | |
| if: steps.binary-cache.outputs.reuse != 'true' | |
| run: rustup target add wasm32-unknown-unknown | |
| - name: Install wasm-bindgen CLI | |
| if: steps.binary-cache.outputs.reuse != 'true' | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: wasm-bindgen-cli@0.2.122 | |
| - name: Install JS SDK dependencies | |
| working-directory: packages/js-sdk | |
| run: npm ci | |
| - name: Test plugin archive packaging | |
| run: node --test scripts/plugin-archive.test.mjs | |
| - name: Install filesystem storage dependencies | |
| if: matrix.runtime == 'native' | |
| working-directory: packages/storage-filesystem | |
| run: npm ci --legacy-peer-deps | |
| - name: Install OPFS storage dependencies | |
| if: matrix.runtime == 'browser' | |
| working-directory: packages/storage-opfs | |
| run: npm ci | |
| - name: Install Chromium | |
| if: matrix.runtime == 'browser' && steps.binary-cache.outputs.reuse == 'true' | |
| working-directory: packages/js-sdk | |
| run: npx playwright install --with-deps chromium | |
| - name: Build JS SDK with exact-input binary reuse | |
| env: | |
| REUSE_BINARIES: ${{ steps.binary-cache.outputs.reuse }} | |
| BINARY_KEY: ${{ steps.binary-key.outputs.key }} | |
| SDK_RUNTIME: ${{ matrix.runtime }} | |
| run: | | |
| npm --prefix packages/js-sdk run clean | |
| if [ "$REUSE_BINARIES" = true ]; then | |
| node scripts/ci-sdk-cache.mjs restore "$SDK_RUNTIME" "$BINARY_KEY" | |
| else | |
| node scripts/ci-build-sdk.mjs "$SDK_RUNTIME" | |
| node scripts/ci-sdk-cache.mjs save "$SDK_RUNTIME" "$BINARY_KEY" | |
| fi | |
| npm --prefix packages/js-sdk run build:ts | |
| - name: Build and check filesystem storage package | |
| if: matrix.runtime == 'native' | |
| working-directory: packages/storage-filesystem | |
| run: | | |
| node scripts/link-sdk.js | |
| npm run build | |
| npm run typecheck | |
| npm test | |
| - name: Typecheck native JS SDK | |
| if: matrix.runtime == 'native' | |
| working-directory: packages/js-sdk | |
| run: npm run typecheck | |
| - name: Run native JS SDK tests | |
| if: matrix.runtime == 'native' | |
| working-directory: packages/js-sdk | |
| run: npm exec -- vitest run | |
| - name: Verify installed native SDK upgrades released repositories | |
| if: matrix.runtime == 'native' | |
| working-directory: packages/js-sdk | |
| run: npm run test:native:production | |
| - name: Build and check OPFS storage package | |
| if: matrix.runtime == 'browser' | |
| working-directory: packages/storage-opfs | |
| run: | | |
| npm run build | |
| npm run typecheck | |
| npm pack --dry-run --ignore-scripts | |
| - name: Run browser integration suites | |
| if: matrix.runtime == 'browser' | |
| run: node scripts/ci-test-browser.mjs | |
| - name: Describe reusable browser SDK artifact | |
| if: matrix.runtime == 'browser' | |
| run: node scripts/release-browser-artifact.mjs describe | |
| - name: Upload browser build provenance for release lookup | |
| if: matrix.runtime == 'browser' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ci-browser-build | |
| path: ci-artifact/browser.json | |
| retention-days: 90 | |
| if-no-files-found: error | |
| - name: Upload tested browser SDK for submodule consumers | |
| if: matrix.runtime == 'browser' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: lix-browser-sdk-${{ env.LIX_SOURCE_SHA }} | |
| path: | | |
| packages/js-sdk/dist | |
| packages/storage-opfs/dist | |
| ci-artifact/browser.json | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Upload SDK build phase timings | |
| if: always() && steps.binary-cache.outputs.reuse != 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdk-build-timings-${{ matrix.runtime }}-${{ github.run_attempt }} | |
| path: ci-sdk-timings/${{ matrix.runtime }} | |
| retention-days: 90 | |
| if-no-files-found: warn | |
| - name: Cache successfully tested SDK binaries | |
| if: steps.binary-cache.outputs.reuse != 'true' && steps.binaries.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: .ci-sdk-cache/${{ matrix.runtime }} | |
| key: ${{ steps.binary-key.outputs.key }} | |
| - name: Report remaining runner resources | |
| if: always() | |
| run: | | |
| df -h "$GITHUB_WORKSPACE" | |
| free -h | |
| if command -v sccache >/dev/null; then sccache --show-stats; fi | |
| preview-artifact-changes: | |
| name: Select preview artifacts | |
| if: (github.event_name == 'pull_request' && github.event.pull_request.draft == false) || github.event_name == 'push' || inputs.artifacts_only == true | |
| needs: merge-reuse | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| outputs: | |
| server: ${{ steps.changes.outputs.server }} | |
| steps: | |
| - name: Checkout feature revision | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: Detect reference server inputs | |
| id: changes | |
| env: | |
| ARTIFACTS_ONLY: ${{ inputs.artifacts_only || false }} | |
| PROMOTE_SERVER: ${{ needs.merge-reuse.outputs.server_artifact }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| if [ "$ARTIFACTS_ONLY" = true ]; then | |
| echo 'server=true' >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [ "$GITHUB_EVENT_NAME" = push ]; then | |
| if [ "$PROMOTE_SERVER" = true ]; then | |
| echo 'server=false' >> "$GITHUB_OUTPUT" | |
| else | |
| echo 'server=true' >> "$GITHUB_OUTPUT" | |
| fi | |
| exit 0 | |
| fi | |
| if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- \ | |
| .github/workflows/ci.yml \ | |
| .github/workflows/publish-packages.yml \ | |
| .cargo \ | |
| .dockerignore \ | |
| Cargo.lock \ | |
| Cargo.toml \ | |
| rust-toolchain.toml \ | |
| packages/lix \ | |
| packages/lix-schema \ | |
| packages/server \ | |
| packages/storage-slatedb \ | |
| plugins; then | |
| echo 'server=false' >> "$GITHUB_OUTPUT" | |
| echo 'Reference server inputs are unchanged; no preview image is needed.' | |
| else | |
| echo 'server=true' >> "$GITHUB_OUTPUT" | |
| echo 'Reference server inputs changed; building one reusable preview image.' | |
| fi | |
| preview-server-image: | |
| name: Lix server preview artifact | |
| if: needs.preview-artifact-changes.outputs.server == 'true' | |
| needs: preview-artifact-changes | |
| runs-on: ubicloud-standard-30-ubuntu-2404 | |
| timeout-minutes: 45 | |
| env: | |
| LIX_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| steps: | |
| - name: Checkout feature revision | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.LIX_SOURCE_SHA }} | |
| - name: Prepare Linux build runner | |
| uses: ./.github/actions/prepare-linux-runner | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build reusable reference server image once | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: packages/server/Dockerfile | |
| build-args: | | |
| LIX_SOURCE_REVISION=${{ env.LIX_SOURCE_SHA }} | |
| tags: lix-server-ci:${{ env.LIX_SOURCE_SHA }} | |
| outputs: type=docker,dest=${{ runner.temp }}/lix-server-image.tar | |
| labels: | | |
| org.opencontainers.image.revision=${{ env.LIX_SOURCE_SHA }} | |
| org.opencontainers.image.source=https://github.com/${{ github.repository }} | |
| cache-from: type=gha,scope=lix-server-preview | |
| cache-to: type=gha,mode=max,scope=lix-server-preview | |
| - name: Verify reusable server image metadata | |
| run: | | |
| docker load --input "$RUNNER_TEMP/lix-server-image.tar" | |
| revision="$(docker image inspect \ | |
| "lix-server-ci:$LIX_SOURCE_SHA" \ | |
| --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" | |
| test "$revision" = "$LIX_SOURCE_SHA" | |
| docker image inspect "lix-server-ci:$LIX_SOURCE_SHA" \ | |
| --format '{{json .Config.Env}}' | node -e ' | |
| let input = ""; | |
| process.stdin.on("data", chunk => input += chunk); | |
| process.stdin.on("end", () => { | |
| if (!JSON.parse(input).includes(`LIX_SOURCE_REVISION=${process.env.LIX_SOURCE_SHA}`)) { | |
| throw new Error("server telemetry source revision is missing or incorrect"); | |
| } | |
| }); | |
| ' | |
| echo "Built reusable Lix server image for $revision" | |
| - name: Describe reusable server image artifact | |
| run: | | |
| mkdir -p ci-artifact | |
| cp "$RUNNER_TEMP/lix-server-image.tar" ci-artifact/lix-server-image.tar | |
| node --input-type=module - <<'EOF' > ci-artifact/server-linux-x64.json | |
| console.log(JSON.stringify({ | |
| schemaVersion: 1, | |
| kind: "lix-server-image", | |
| sourceRevision: process.env.LIX_SOURCE_SHA, | |
| target: "linux-x64", | |
| image: `lix-server-ci:${process.env.LIX_SOURCE_SHA}`, | |
| }, null, 2)); | |
| EOF | |
| - name: Upload tested server image for preview consumers | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: lix-server-image-linux-x64-${{ env.LIX_SOURCE_SHA }} | |
| path: | | |
| ci-artifact/lix-server-image.tar | |
| ci-artifact/server-linux-x64.json | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Report remaining runner resources | |
| if: always() | |
| run: | | |
| df -h "$GITHUB_WORKSPACE" | |
| free -h | |
| if command -v sccache >/dev/null; then sccache --show-stats; fi | |
| content-browser-sdk: | |
| name: Reuse unchanged content SDK | |
| needs: merge-reuse | |
| if: needs.merge-reuse.outputs.content_only == 'true' && needs.merge-reuse.outputs.reuse != 'true' | |
| runs-on: ubicloud-standard-2-ubuntu-2404 | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| TARGET_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} | |
| CARGO_INCREMENTAL: '0' | |
| CARGO_PROFILE_DEV_DEBUG: '0' | |
| CARGO_PROFILE_TEST_DEBUG: '0' | |
| CARGO_PROFILE_RELEASE_DEBUG: '0' | |
| CARGO_PROFILE_BENCH_DEBUG: '0' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ env.TARGET_REVISION }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| - name: Find tested SDK with identical code inputs | |
| id: source | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { selectContentArtifact } = await import(`${process.env.GITHUB_WORKSPACE}/scripts/ci-content-artifact.mjs`); | |
| await selectContentArtifact({ github, context, core }); | |
| - uses: actions/download-artifact@v4 | |
| if: steps.source.outputs.run_id != '' | |
| with: | |
| name: lix-browser-sdk-${{ steps.source.outputs.revision }} | |
| run-id: ${{ steps.source.outputs.run_id }} | |
| github-token: ${{ github.token }} | |
| - name: Verify and promote unchanged SDK | |
| if: steps.source.outputs.run_id != '' | |
| env: | |
| SOURCE_REVISION: ${{ steps.source.outputs.revision }} | |
| SOURCE_RUN: ${{ steps.source.outputs.run_id }} | |
| run: node scripts/ci-content-artifact.mjs | |
| - uses: actions/upload-artifact@v4 | |
| if: steps.source.outputs.run_id != '' | |
| with: | |
| name: lix-browser-sdk-${{ env.TARGET_REVISION }} | |
| path: | | |
| packages/js-sdk/dist | |
| packages/storage-opfs/dist | |
| ci-artifact/browser.json | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - uses: actions/upload-artifact@v4 | |
| if: steps.source.outputs.run_id != '' | |
| with: | |
| name: ci-browser-build | |
| path: ci-artifact/browser.json | |
| retention-days: 90 | |
| if-no-files-found: error |