From 1c735cc840779276cfb29e9ed7331809cc61a9a0 Mon Sep 17 00:00:00 2001 From: Valmo Date: Tue, 14 Jul 2026 15:56:24 -0300 Subject: [PATCH] Optimize DEV workshop deployment cache --- .gitea/workflows/dev-workshop.yml | 221 +++++++++++++++++++----------- make/ci_dev_workshop.py | 48 ++++++- 2 files changed, 187 insertions(+), 82 deletions(-) diff --git a/.gitea/workflows/dev-workshop.yml b/.gitea/workflows/dev-workshop.yml index e25e6c6..8988cb7 100644 --- a/.gitea/workflows/dev-workshop.yml +++ b/.gitea/workflows/dev-workshop.yml @@ -7,7 +7,8 @@ on: env: STEAM_APP_ID: "107410" - BRAF_CI_CACHE_DIR: .ci-cache/dev-workshop + HOST_REPO: /home/valmo/braf + HOST_CACHE: /home/valmo/braf-ci-cache/dev-workshop BRAF_CI_STAGE_DIR: dist/dev/@braf_dev jobs: @@ -15,102 +16,164 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 2 - lfs: true - - - name: Restore PBO cache - uses: actions/cache@v4 - with: - path: .ci-cache/dev-workshop - key: braf-dev-workshop-${{ github.sha }} - restore-keys: | - braf-dev-workshop- - - - name: Install Mikero tools + - name: Build changed addons and upload DEV Workshop item shell: bash + env: + GIT_SHA: ${{ github.sha }} + GIT_BEFORE: ${{ github.event.before }} + STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }} + STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }} + STEAM_DEV_ITEM_ID: ${{ secrets.STEAM_DEV_ITEM_ID }} run: | set -euo pipefail - sudo apt-get update - sudo apt-get install -y libogg0 libuchardet0 libvorbis0a libvorbisenc2 libvorbisfile3 + mkdir -p "$HOST_CACHE" - curl -fsSL \ - https://github.com/arma-actions/mikero-tools/archive/refs/heads/master.tar.gz \ - -o /tmp/mikero-tools-action.tar.gz - - rm -rf /tmp/mikero-tools-action /opt/mikero-tools - mkdir -p /tmp/mikero-tools-action /opt/mikero-tools - tar -zxf /tmp/mikero-tools-action.tar.gz \ - --strip-components=1 \ - -C /tmp/mikero-tools-action - - tar -xf /tmp/mikero-tools-action/linux/*.tar \ - --strip-components=1 \ - -C /opt/mikero-tools - - echo "/opt/mikero-tools/bin" >> "$GITHUB_PATH" - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/opt/mikero-tools/lib" >> "$GITHUB_ENV" - - - name: Detect global build changes - id: changes - shell: bash - run: | + docker run --rm -i \ + -e BRAF_CI_STAGE_DIR \ + -e GIT_BEFORE \ + -e GIT_SHA \ + -e HOST_CACHE \ + -e HOST_REPO \ + -e STEAM_APP_ID \ + -e STEAM_DEV_ITEM_ID \ + -e STEAM_PASSWORD \ + -e STEAM_USERNAME \ + -v "$HOST_REPO:/repo" \ + -v "$HOST_CACHE:/cache" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker.gitea.com/runner-images:ubuntu-latest \ + bash -s <<'BRAF_CI' set -euo pipefail - before="${{ github.event.before }}" - after="${{ github.sha }}" + install_runtime() { + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y \ + ca-certificates \ + curl \ + git-lfs \ + libogg0 \ + libuchardet0 \ + libvorbis0a \ + libvorbisenc2 \ + libvorbisfile3 + } - if [[ -z "${before}" || "${before}" == "0000000000000000000000000000000000000000" ]]; then - echo "force_rebuild=true" >> "$GITHUB_OUTPUT" + install_mikero() { + if [[ ! -x /cache/mikero-tools/bin/makepbo ]]; then + rm -rf /tmp/mikero-tools-action /cache/mikero-tools + mkdir -p /tmp/mikero-tools-action /cache/mikero-tools + curl -fsSL \ + https://github.com/arma-actions/mikero-tools/archive/refs/heads/master.tar.gz \ + -o /tmp/mikero-tools-action.tar.gz + tar -zxf /tmp/mikero-tools-action.tar.gz \ + --strip-components=1 \ + -C /tmp/mikero-tools-action + tar -xf /tmp/mikero-tools-action/linux/*.tar \ + --strip-components=1 \ + -C /cache/mikero-tools + fi + + export PATH="/cache/mikero-tools/bin:$PATH" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/cache/mikero-tools/lib" + } + + is_zero_sha() { + [[ -z "${1:-}" || "$1" == "0000000000000000000000000000000000000000" ]] + } + + addon_csv() { + local IFS=, + echo "$*" + } + + install_runtime + install_mikero + + cd /repo + git config --global --add safe.directory /repo + git lfs install --local + git fetch --no-tags origin "$GIT_SHA" || git fetch --no-tags origin main + + if is_zero_sha "$GIT_BEFORE" || ! git cat-file -e "$GIT_BEFORE^{commit}" 2>/dev/null; then + mapfile -t changed_addons < <( + git ls-tree -d --name-only "$GIT_SHA" | + awk '/^braf_/ {print}' | + while read -r addon; do + if git cat-file -e "$GIT_SHA:$addon/config.cpp" 2>/dev/null; then + printf '%s\n' "$addon" + fi + done + ) + metadata_changed=true + else + mapfile -t changed_files < <(git diff --name-only "$GIT_BEFORE" "$GIT_SHA") + mapfile -t changed_addons < <( + printf '%s\n' "${changed_files[@]}" | + awk -F/ '$1 ~ /^braf_/ {print $1}' | + sort -u | + while read -r addon; do + if git cat-file -e "$GIT_SHA:$addon/config.cpp" 2>/dev/null; then + printf '%s\n' "$addon" + fi + done + ) + + metadata_changed=false + if printf '%s\n' "${changed_files[@]}" | grep -Eq '^(mod\.cpp|meta\.cpp)$'; then + metadata_changed=true + fi + fi + + if [[ "${#changed_addons[@]}" -eq 0 && "$metadata_changed" != "true" ]]; then + echo "No addon or workshop metadata changes detected; skipping DEV Workshop deploy." exit 0 fi - changed="$(git diff --name-only "${before}" "${after}" || printf '%s\n' '__force_rebuild__')" - printf '%s\n' "${changed}" + GIT_LFS_SKIP_SMUDGE=1 git reset --hard "$GIT_SHA" - if printf '%s\n' "${changed}" | grep -Eq '^(__force_rebuild__|\.gitea/workflows/|make/|mod\.cpp$|meta\.cpp$)'; then - echo "force_rebuild=true" >> "$GITHUB_OUTPUT" + if [[ "${#changed_addons[@]}" -gt 0 ]]; then + include="" + for addon in "${changed_addons[@]}"; do + include+="${include:+,}${addon}/**" + done + + echo "Changed addons: $(addon_csv "${changed_addons[@]}")" + echo "Fetching LFS only for: $include" + git lfs fetch origin "$GIT_SHA" --include="$include" --exclude="*" + git lfs checkout "${changed_addons[@]}" else - echo "force_rebuild=false" >> "$GITHUB_OUTPUT" + echo "Only workshop metadata changed; using cached PBOs." fi - - name: Build and stage DEV package - shell: bash - run: | - set -euo pipefail - - mkdir -p "$BRAF_CI_CACHE_DIR" - - args=( - --cache-dir "$BRAF_CI_CACHE_DIR" + build_args=( + --cache-dir /cache/pbo --stage-dir "$BRAF_CI_STAGE_DIR" + --addons "$(addon_csv "${changed_addons[@]}")" ) - if [[ "${{ steps.changes.outputs.force_rebuild }}" == "true" ]]; then - args+=(--force-rebuild) - fi + python3 -m make.ci_dev_workshop "${build_args[@]}" - python3 -m make.ci_dev_workshop "${args[@]}" - - - name: Prepare changelog - id: changelog - shell: bash - run: | - set -euo pipefail - short_sha="$(git rev-parse --short HEAD)" + short_sha="$(git rev-parse --short "$GIT_SHA")" subject="$(git log -1 --pretty=%s)" - echo "text=DEV ${short_sha}: ${subject}" >> "$GITHUB_OUTPUT" + changelog="DEV ${short_sha}: ${subject}" - - name: Upload DEV Workshop item - uses: https://github.com/arma-actions/workshop-upload@v1 - env: - STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }} - STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }} - with: - appId: ${{ env.STEAM_APP_ID }} - itemId: ${{ secrets.STEAM_DEV_ITEM_ID }} - contentPath: ${{ env.BRAF_CI_STAGE_DIR }} - changelog: ${{ steps.changelog.outputs.text }} + cat > /cache/braf-dev-workshop.vdf < argparse.Namespace: action="store_true", help="Rebuild every buildable addon, ignoring matching cache hashes.", ) + parser.add_argument( + "--addons", + default=None, + help=( + "Comma-separated addon allowlist to rebuild. When set, only these " + "addons may be rebuilt; unchanged addons are staged from cache." + ), + ) parser.add_argument( "--dry-run", action="store_true", @@ -78,6 +86,14 @@ def buildable_addons(root: Path) -> list[str]: return addons +def parse_addon_allowlist(value: str | None) -> list[str] | None: + if value is None: + return None + + addons = [part.strip() for part in value.split(",") if part.strip()] + return sorted(dict.fromkeys(addons)) + + def file_digest(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -289,12 +305,24 @@ def main() -> int: stage_dir = (root / args.stage_dir).resolve() addons = buildable_addons(root) + requested_addons = parse_addon_allowlist(args.addons) + if requested_addons is not None: + unknown_addons = sorted(set(requested_addons) - set(addons)) + if unknown_addons: + print( + "ERROR: Requested addon is not buildable: " + + ", ".join(unknown_addons), + file=sys.stderr, + ) + return 2 + if args.source_root: source_root = Path(args.source_root).resolve() elif args.dry_run: source_root = (cache_dir / "mikero-source" / "braf").resolve() else: - source_root = prepare_mikero_source_root(root, cache_dir, addons).resolve() + source_root_addons = requested_addons if requested_addons is not None else addons + source_root = prepare_mikero_source_root(root, cache_dir, source_root_addons).resolve() current_hashes = {addon: addon_digest(root / addon) for addon in addons} manifest = load_manifest(manifest_path) @@ -308,7 +336,8 @@ def main() -> int: manifest_addons.pop(addon, None) to_build: list[str] = [] - for addon in addons: + candidate_addons = requested_addons if requested_addons is not None else addons + for addon in candidate_addons: cached_hash = manifest_addons.get(addon, {}).get("hash") if ( args.force_rebuild @@ -318,6 +347,11 @@ def main() -> int: to_build.append(addon) print(f"Buildable addons: {len(addons)}") + if requested_addons is not None: + print( + "Requested rebuild allowlist: " + + (", ".join(requested_addons) if requested_addons else "none") + ) print(f"Addons to build: {', '.join(to_build) if to_build else 'none'}") print(f"Cache dir: {cache_dir}") print(f"Stage dir: {stage_dir}") @@ -336,7 +370,15 @@ def main() -> int: for addon in addons: if not cached_pbo_exists(cache_addons_dir, addon): - raise RuntimeError(f"Cache does not contain required PBO after build: {addon}") + hint = ( + " Seed the persistent cache first or include this addon in the " + "detected rebuild set." + if requested_addons is not None and addon not in candidate_addons + else "" + ) + raise RuntimeError( + f"Cache does not contain required PBO after build: {addon}.{hint}" + ) stage_package(root, stage_dir, cache_addons_dir, addons) write_manifest(manifest_path, manifest)