Optimize DEV workshop deployment cache

.gitea/workflows/dev-workshop.yml

@@ -7,7 +7,8 @@ on:
 
 env:
   STEAM_APP_ID: "107410"
-  BRAF_CI_CACHE_DIR: .ci-cache/dev-workshop
+  HOST_REPO: /home/valmo/braf
+  HOST_CACHE: /home/valmo/braf-ci-cache/dev-workshop
   BRAF_CI_STAGE_DIR: dist/dev/@braf_dev
 
 jobs:
@@ -15,102 +16,164 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout
+      - name: Build changed addons and upload DEV Workshop item
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-          lfs: true
-
-      - name: Restore PBO cache
-        uses: actions/cache@v4
-        with:
-          path: .ci-cache/dev-workshop
-          key: braf-dev-workshop-${{ github.sha }}
-          restore-keys: |
-            braf-dev-workshop-
-
-      - name: Install Mikero tools
         shell: bash
+        env:
+          GIT_SHA: ${{ github.sha }}
+          GIT_BEFORE: ${{ github.event.before }}
+          STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+          STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
+          STEAM_DEV_ITEM_ID: ${{ secrets.STEAM_DEV_ITEM_ID }}
         run: |
           set -euo pipefail
 
-          sudo apt-get update
+          mkdir -p "$HOST_CACHE"
-          sudo apt-get install -y libogg0 libuchardet0 libvorbis0a libvorbisenc2 libvorbisfile3
 
+          docker run --rm -i \
+            -e BRAF_CI_STAGE_DIR \
+            -e GIT_BEFORE \
+            -e GIT_SHA \
+            -e HOST_CACHE \
+            -e HOST_REPO \
+            -e STEAM_APP_ID \
+            -e STEAM_DEV_ITEM_ID \
+            -e STEAM_PASSWORD \
+            -e STEAM_USERNAME \
+            -v "$HOST_REPO:/repo" \
+            -v "$HOST_CACHE:/cache" \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            docker.gitea.com/runner-images:ubuntu-latest \
+            bash -s <<'BRAF_CI'
+          set -euo pipefail
+
+          install_runtime() {
+            export DEBIAN_FRONTEND=noninteractive
+            apt-get update
+            apt-get install -y \
+              ca-certificates \
+              curl \
+              git-lfs \
+              libogg0 \
+              libuchardet0 \
+              libvorbis0a \
+              libvorbisenc2 \
+              libvorbisfile3
+          }
+
+          install_mikero() {
+            if [[ ! -x /cache/mikero-tools/bin/makepbo ]]; then
+              rm -rf /tmp/mikero-tools-action /cache/mikero-tools
+              mkdir -p /tmp/mikero-tools-action /cache/mikero-tools
               curl -fsSL \
                 https://github.com/arma-actions/mikero-tools/archive/refs/heads/master.tar.gz \
                 -o /tmp/mikero-tools-action.tar.gz
-
-          rm -rf /tmp/mikero-tools-action /opt/mikero-tools
-          mkdir -p /tmp/mikero-tools-action /opt/mikero-tools
               tar -zxf /tmp/mikero-tools-action.tar.gz \
                 --strip-components=1 \
                 -C /tmp/mikero-tools-action
-
               tar -xf /tmp/mikero-tools-action/linux/*.tar \
                 --strip-components=1 \
-            -C /opt/mikero-tools
+                -C /cache/mikero-tools
+            fi
 
-          echo "/opt/mikero-tools/bin" >> "$GITHUB_PATH"
+            export PATH="/cache/mikero-tools/bin:$PATH"
-          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/opt/mikero-tools/lib" >> "$GITHUB_ENV"
+            export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/cache/mikero-tools/lib"
+          }
 
-      - name: Detect global build changes
+          is_zero_sha() {
-        id: changes
+            [[ -z "${1:-}" || "$1" == "0000000000000000000000000000000000000000" ]]
-        shell: bash
+          }
-        run: |
-          set -euo pipefail
 
-          before="${{ github.event.before }}"
+          addon_csv() {
-          after="${{ github.sha }}"
+            local IFS=,
+            echo "$*"
+          }
 
-          if [[ -z "${before}" || "${before}" == "0000000000000000000000000000000000000000" ]]; then
+          install_runtime
-            echo "force_rebuild=true" >> "$GITHUB_OUTPUT"
+          install_mikero
+
+          cd /repo
+          git config --global --add safe.directory /repo
+          git lfs install --local
+          git fetch --no-tags origin "$GIT_SHA" || git fetch --no-tags origin main
+
+          if is_zero_sha "$GIT_BEFORE" || ! git cat-file -e "$GIT_BEFORE^{commit}" 2>/dev/null; then
+            mapfile -t changed_addons < <(
+              git ls-tree -d --name-only "$GIT_SHA" |
+                awk '/^braf_/ {print}' |
+                while read -r addon; do
+                  if git cat-file -e "$GIT_SHA:$addon/config.cpp" 2>/dev/null; then
+                    printf '%s\n' "$addon"
+                  fi
+                done
+            )
+            metadata_changed=true
+          else
+            mapfile -t changed_files < <(git diff --name-only "$GIT_BEFORE" "$GIT_SHA")
+            mapfile -t changed_addons < <(
+              printf '%s\n' "${changed_files[@]}" |
+                awk -F/ '$1 ~ /^braf_/ {print $1}' |
+                sort -u |
+                while read -r addon; do
+                  if git cat-file -e "$GIT_SHA:$addon/config.cpp" 2>/dev/null; then
+                    printf '%s\n' "$addon"
+                  fi
+                done
+            )
+
+            metadata_changed=false
+            if printf '%s\n' "${changed_files[@]}" | grep -Eq '^(mod\.cpp|meta\.cpp)$'; then
+              metadata_changed=true
+            fi
+          fi
+
+          if [[ "${#changed_addons[@]}" -eq 0 && "$metadata_changed" != "true" ]]; then
+            echo "No addon or workshop metadata changes detected; skipping DEV Workshop deploy."
             exit 0
           fi
 
-          changed="$(git diff --name-only "${before}" "${after}" || printf '%s\n' '__force_rebuild__')"
+          GIT_LFS_SKIP_SMUDGE=1 git reset --hard "$GIT_SHA"
-          printf '%s\n' "${changed}"
 
-          if printf '%s\n' "${changed}" | grep -Eq '^(__force_rebuild__|\.gitea/workflows/|make/|mod\.cpp$|meta\.cpp$)'; then
+          if [[ "${#changed_addons[@]}" -gt 0 ]]; then
-            echo "force_rebuild=true" >> "$GITHUB_OUTPUT"
+            include=""
+            for addon in "${changed_addons[@]}"; do
+              include+="${include:+,}${addon}/**"
+            done
+
+            echo "Changed addons: $(addon_csv "${changed_addons[@]}")"
+            echo "Fetching LFS only for: $include"
+            git lfs fetch origin "$GIT_SHA" --include="$include" --exclude="*"
+            git lfs checkout "${changed_addons[@]}"
           else
-            echo "force_rebuild=false" >> "$GITHUB_OUTPUT"
+            echo "Only workshop metadata changed; using cached PBOs."
           fi
 
-      - name: Build and stage DEV package
+          build_args=(
-        shell: bash
+            --cache-dir /cache/pbo
-        run: |
-          set -euo pipefail
-
-          mkdir -p "$BRAF_CI_CACHE_DIR"
-
-          args=(
-            --cache-dir "$BRAF_CI_CACHE_DIR"
             --stage-dir "$BRAF_CI_STAGE_DIR"
+            --addons "$(addon_csv "${changed_addons[@]}")"
           )
 
-          if [[ "${{ steps.changes.outputs.force_rebuild }}" == "true" ]]; then
+          python3 -m make.ci_dev_workshop "${build_args[@]}"
-            args+=(--force-rebuild)
-          fi
 
-          python3 -m make.ci_dev_workshop "${args[@]}"
+          short_sha="$(git rev-parse --short "$GIT_SHA")"
-
-      - name: Prepare changelog
-        id: changelog
-        shell: bash
-        run: |
-          set -euo pipefail
-          short_sha="$(git rev-parse --short HEAD)"
           subject="$(git log -1 --pretty=%s)"
-          echo "text=DEV ${short_sha}: ${subject}" >> "$GITHUB_OUTPUT"
+          changelog="DEV ${short_sha}: ${subject}"
 
-      - name: Upload DEV Workshop item
+          cat > /cache/braf-dev-workshop.vdf <<EOF
-        uses: https://github.com/arma-actions/workshop-upload@v1
+          "workshopitem"
-        env:
+          {
-          STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+            "appid" "$STEAM_APP_ID"
-          STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
+            "publishedfileid" "$STEAM_DEV_ITEM_ID"
-        with:
+            "contentfolder" "/content"
-          appId: ${{ env.STEAM_APP_ID }}
+            "changenote" "$changelog"
-          itemId: ${{ secrets.STEAM_DEV_ITEM_ID }}
+          }
-          contentPath: ${{ env.BRAF_CI_STAGE_DIR }}
+          EOF
-          changelog: ${{ steps.changelog.outputs.text }}
+
+          docker run --rm \
+            -e STEAM_PASSWORD \
+            -e STEAM_USERNAME \
+            -v "${HOST_REPO}/${BRAF_CI_STAGE_DIR}:/content:ro" \
+            -v "${HOST_CACHE}/braf-dev-workshop.vdf:/tmp/braf-dev-workshop.vdf:ro" \
+            cm2network/steamcmd:root \
+            bash -lc 'steamcmd +login "$STEAM_USERNAME" "$STEAM_PASSWORD" +workshop_build_item /tmp/braf-dev-workshop.vdf +quit'
+          BRAF_CI

make/ci_dev_workshop.py

@@ -58,6 +58,14 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Rebuild every buildable addon, ignoring matching cache hashes.",
     )
+    parser.add_argument(
+        "--addons",
+        default=None,
+        help=(
+            "Comma-separated addon allowlist to rebuild. When set, only these "
+            "addons may be rebuilt; unchanged addons are staged from cache."
+        ),
+    )
     parser.add_argument(
         "--dry-run",
         action="store_true",
@@ -78,6 +86,14 @@ def buildable_addons(root: Path) -> list[str]:
     return addons
 
 
+def parse_addon_allowlist(value: str | None) -> list[str] | None:
+    if value is None:
+        return None
+
+    addons = [part.strip() for part in value.split(",") if part.strip()]
+    return sorted(dict.fromkeys(addons))
+
+
 def file_digest(path: Path) -> str:
     digest = hashlib.sha256()
     with path.open("rb") as handle:
@@ -289,12 +305,24 @@ def main() -> int:
     stage_dir = (root / args.stage_dir).resolve()
 
     addons = buildable_addons(root)
+    requested_addons = parse_addon_allowlist(args.addons)
+    if requested_addons is not None:
+        unknown_addons = sorted(set(requested_addons) - set(addons))
+        if unknown_addons:
+            print(
+                "ERROR: Requested addon is not buildable: "
+                + ", ".join(unknown_addons),
+                file=sys.stderr,
+            )
+            return 2
+
     if args.source_root:
         source_root = Path(args.source_root).resolve()
     elif args.dry_run:
         source_root = (cache_dir / "mikero-source" / "braf").resolve()
     else:
-        source_root = prepare_mikero_source_root(root, cache_dir, addons).resolve()
+        source_root_addons = requested_addons if requested_addons is not None else addons
+        source_root = prepare_mikero_source_root(root, cache_dir, source_root_addons).resolve()
 
     current_hashes = {addon: addon_digest(root / addon) for addon in addons}
     manifest = load_manifest(manifest_path)
@@ -308,7 +336,8 @@ def main() -> int:
             manifest_addons.pop(addon, None)
 
     to_build: list[str] = []
-    for addon in addons:
+    candidate_addons = requested_addons if requested_addons is not None else addons
+    for addon in candidate_addons:
         cached_hash = manifest_addons.get(addon, {}).get("hash")
         if (
             args.force_rebuild
@@ -318,6 +347,11 @@ def main() -> int:
             to_build.append(addon)
 
     print(f"Buildable addons: {len(addons)}")
+    if requested_addons is not None:
+        print(
+            "Requested rebuild allowlist: "
+            + (", ".join(requested_addons) if requested_addons else "none")
+        )
     print(f"Addons to build: {', '.join(to_build) if to_build else 'none'}")
     print(f"Cache dir: {cache_dir}")
     print(f"Stage dir: {stage_dir}")
@@ -336,7 +370,15 @@ def main() -> int:
 
     for addon in addons:
         if not cached_pbo_exists(cache_addons_dir, addon):
-            raise RuntimeError(f"Cache does not contain required PBO after build: {addon}")
+            hint = (
+                " Seed the persistent cache first or include this addon in the "
+                "detected rebuild set."
+                if requested_addons is not None and addon not in candidate_addons
+                else ""
+            )
+            raise RuntimeError(
+                f"Cache does not contain required PBO after build: {addon}.{hint}"
+            )
 
     stage_package(root, stage_dir, cache_addons_dir, addons)
     write_manifest(manifest_path, manifest)