Optimize DEV workshop deployment cache
This commit is contained in:
@@ -7,7 +7,8 @@ on:
|
||||
|
||||
env:
|
||||
STEAM_APP_ID: "107410"
|
||||
BRAF_CI_CACHE_DIR: .ci-cache/dev-workshop
|
||||
HOST_REPO: /home/valmo/braf
|
||||
HOST_CACHE: /home/valmo/braf-ci-cache/dev-workshop
|
||||
BRAF_CI_STAGE_DIR: dist/dev/@braf_dev
|
||||
|
||||
jobs:
|
||||
@@ -15,102 +16,164 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
lfs: true
|
||||
|
||||
- name: Restore PBO cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: .ci-cache/dev-workshop
|
||||
key: braf-dev-workshop-${{ github.sha }}
|
||||
restore-keys: |
|
||||
braf-dev-workshop-
|
||||
|
||||
- name: Install Mikero tools
|
||||
- name: Build changed addons and upload DEV Workshop item
|
||||
shell: bash
|
||||
env:
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
GIT_BEFORE: ${{ github.event.before }}
|
||||
STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
|
||||
STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
|
||||
STEAM_DEV_ITEM_ID: ${{ secrets.STEAM_DEV_ITEM_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libogg0 libuchardet0 libvorbis0a libvorbisenc2 libvorbisfile3
|
||||
mkdir -p "$HOST_CACHE"
|
||||
|
||||
curl -fsSL \
|
||||
https://github.com/arma-actions/mikero-tools/archive/refs/heads/master.tar.gz \
|
||||
-o /tmp/mikero-tools-action.tar.gz
|
||||
|
||||
rm -rf /tmp/mikero-tools-action /opt/mikero-tools
|
||||
mkdir -p /tmp/mikero-tools-action /opt/mikero-tools
|
||||
tar -zxf /tmp/mikero-tools-action.tar.gz \
|
||||
--strip-components=1 \
|
||||
-C /tmp/mikero-tools-action
|
||||
|
||||
tar -xf /tmp/mikero-tools-action/linux/*.tar \
|
||||
--strip-components=1 \
|
||||
-C /opt/mikero-tools
|
||||
|
||||
echo "/opt/mikero-tools/bin" >> "$GITHUB_PATH"
|
||||
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/opt/mikero-tools/lib" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Detect global build changes
|
||||
id: changes
|
||||
shell: bash
|
||||
run: |
|
||||
docker run --rm -i \
|
||||
-e BRAF_CI_STAGE_DIR \
|
||||
-e GIT_BEFORE \
|
||||
-e GIT_SHA \
|
||||
-e HOST_CACHE \
|
||||
-e HOST_REPO \
|
||||
-e STEAM_APP_ID \
|
||||
-e STEAM_DEV_ITEM_ID \
|
||||
-e STEAM_PASSWORD \
|
||||
-e STEAM_USERNAME \
|
||||
-v "$HOST_REPO:/repo" \
|
||||
-v "$HOST_CACHE:/cache" \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
docker.gitea.com/runner-images:ubuntu-latest \
|
||||
bash -s <<'BRAF_CI'
|
||||
set -euo pipefail
|
||||
|
||||
before="${{ github.event.before }}"
|
||||
after="${{ github.sha }}"
|
||||
install_runtime() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git-lfs \
|
||||
libogg0 \
|
||||
libuchardet0 \
|
||||
libvorbis0a \
|
||||
libvorbisenc2 \
|
||||
libvorbisfile3
|
||||
}
|
||||
|
||||
if [[ -z "${before}" || "${before}" == "0000000000000000000000000000000000000000" ]]; then
|
||||
echo "force_rebuild=true" >> "$GITHUB_OUTPUT"
|
||||
install_mikero() {
|
||||
if [[ ! -x /cache/mikero-tools/bin/makepbo ]]; then
|
||||
rm -rf /tmp/mikero-tools-action /cache/mikero-tools
|
||||
mkdir -p /tmp/mikero-tools-action /cache/mikero-tools
|
||||
curl -fsSL \
|
||||
https://github.com/arma-actions/mikero-tools/archive/refs/heads/master.tar.gz \
|
||||
-o /tmp/mikero-tools-action.tar.gz
|
||||
tar -zxf /tmp/mikero-tools-action.tar.gz \
|
||||
--strip-components=1 \
|
||||
-C /tmp/mikero-tools-action
|
||||
tar -xf /tmp/mikero-tools-action/linux/*.tar \
|
||||
--strip-components=1 \
|
||||
-C /cache/mikero-tools
|
||||
fi
|
||||
|
||||
export PATH="/cache/mikero-tools/bin:$PATH"
|
||||
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/cache/mikero-tools/lib"
|
||||
}
|
||||
|
||||
is_zero_sha() {
|
||||
[[ -z "${1:-}" || "$1" == "0000000000000000000000000000000000000000" ]]
|
||||
}
|
||||
|
||||
addon_csv() {
|
||||
local IFS=,
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
install_runtime
|
||||
install_mikero
|
||||
|
||||
cd /repo
|
||||
git config --global --add safe.directory /repo
|
||||
git lfs install --local
|
||||
git fetch --no-tags origin "$GIT_SHA" || git fetch --no-tags origin main
|
||||
|
||||
if is_zero_sha "$GIT_BEFORE" || ! git cat-file -e "$GIT_BEFORE^{commit}" 2>/dev/null; then
|
||||
mapfile -t changed_addons < <(
|
||||
git ls-tree -d --name-only "$GIT_SHA" |
|
||||
awk '/^braf_/ {print}' |
|
||||
while read -r addon; do
|
||||
if git cat-file -e "$GIT_SHA:$addon/config.cpp" 2>/dev/null; then
|
||||
printf '%s\n' "$addon"
|
||||
fi
|
||||
done
|
||||
)
|
||||
metadata_changed=true
|
||||
else
|
||||
mapfile -t changed_files < <(git diff --name-only "$GIT_BEFORE" "$GIT_SHA")
|
||||
mapfile -t changed_addons < <(
|
||||
printf '%s\n' "${changed_files[@]}" |
|
||||
awk -F/ '$1 ~ /^braf_/ {print $1}' |
|
||||
sort -u |
|
||||
while read -r addon; do
|
||||
if git cat-file -e "$GIT_SHA:$addon/config.cpp" 2>/dev/null; then
|
||||
printf '%s\n' "$addon"
|
||||
fi
|
||||
done
|
||||
)
|
||||
|
||||
metadata_changed=false
|
||||
if printf '%s\n' "${changed_files[@]}" | grep -Eq '^(mod\.cpp|meta\.cpp)$'; then
|
||||
metadata_changed=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${#changed_addons[@]}" -eq 0 && "$metadata_changed" != "true" ]]; then
|
||||
echo "No addon or workshop metadata changes detected; skipping DEV Workshop deploy."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
changed="$(git diff --name-only "${before}" "${after}" || printf '%s\n' '__force_rebuild__')"
|
||||
printf '%s\n' "${changed}"
|
||||
GIT_LFS_SKIP_SMUDGE=1 git reset --hard "$GIT_SHA"
|
||||
|
||||
if printf '%s\n' "${changed}" | grep -Eq '^(__force_rebuild__|\.gitea/workflows/|make/|mod\.cpp$|meta\.cpp$)'; then
|
||||
echo "force_rebuild=true" >> "$GITHUB_OUTPUT"
|
||||
if [[ "${#changed_addons[@]}" -gt 0 ]]; then
|
||||
include=""
|
||||
for addon in "${changed_addons[@]}"; do
|
||||
include+="${include:+,}${addon}/**"
|
||||
done
|
||||
|
||||
echo "Changed addons: $(addon_csv "${changed_addons[@]}")"
|
||||
echo "Fetching LFS only for: $include"
|
||||
git lfs fetch origin "$GIT_SHA" --include="$include" --exclude="*"
|
||||
git lfs checkout "${changed_addons[@]}"
|
||||
else
|
||||
echo "force_rebuild=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Only workshop metadata changed; using cached PBOs."
|
||||
fi
|
||||
|
||||
- name: Build and stage DEV package
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p "$BRAF_CI_CACHE_DIR"
|
||||
|
||||
args=(
|
||||
--cache-dir "$BRAF_CI_CACHE_DIR"
|
||||
build_args=(
|
||||
--cache-dir /cache/pbo
|
||||
--stage-dir "$BRAF_CI_STAGE_DIR"
|
||||
--addons "$(addon_csv "${changed_addons[@]}")"
|
||||
)
|
||||
|
||||
if [[ "${{ steps.changes.outputs.force_rebuild }}" == "true" ]]; then
|
||||
args+=(--force-rebuild)
|
||||
fi
|
||||
python3 -m make.ci_dev_workshop "${build_args[@]}"
|
||||
|
||||
python3 -m make.ci_dev_workshop "${args[@]}"
|
||||
|
||||
- name: Prepare changelog
|
||||
id: changelog
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
short_sha="$(git rev-parse --short HEAD)"
|
||||
short_sha="$(git rev-parse --short "$GIT_SHA")"
|
||||
subject="$(git log -1 --pretty=%s)"
|
||||
echo "text=DEV ${short_sha}: ${subject}" >> "$GITHUB_OUTPUT"
|
||||
changelog="DEV ${short_sha}: ${subject}"
|
||||
|
||||
- name: Upload DEV Workshop item
|
||||
uses: https://github.com/arma-actions/workshop-upload@v1
|
||||
env:
|
||||
STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
|
||||
STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
|
||||
with:
|
||||
appId: ${{ env.STEAM_APP_ID }}
|
||||
itemId: ${{ secrets.STEAM_DEV_ITEM_ID }}
|
||||
contentPath: ${{ env.BRAF_CI_STAGE_DIR }}
|
||||
changelog: ${{ steps.changelog.outputs.text }}
|
||||
cat > /cache/braf-dev-workshop.vdf <<EOF
|
||||
"workshopitem"
|
||||
{
|
||||
"appid" "$STEAM_APP_ID"
|
||||
"publishedfileid" "$STEAM_DEV_ITEM_ID"
|
||||
"contentfolder" "/content"
|
||||
"changenote" "$changelog"
|
||||
}
|
||||
EOF
|
||||
|
||||
docker run --rm \
|
||||
-e STEAM_PASSWORD \
|
||||
-e STEAM_USERNAME \
|
||||
-v "${HOST_REPO}/${BRAF_CI_STAGE_DIR}:/content:ro" \
|
||||
-v "${HOST_CACHE}/braf-dev-workshop.vdf:/tmp/braf-dev-workshop.vdf:ro" \
|
||||
cm2network/steamcmd:root \
|
||||
bash -lc 'steamcmd +login "$STEAM_USERNAME" "$STEAM_PASSWORD" +workshop_build_item /tmp/braf-dev-workshop.vdf +quit'
|
||||
BRAF_CI
|
||||
|
||||
+45
-3
@@ -58,6 +58,14 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Rebuild every buildable addon, ignoring matching cache hashes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--addons",
|
||||
default=None,
|
||||
help=(
|
||||
"Comma-separated addon allowlist to rebuild. When set, only these "
|
||||
"addons may be rebuilt; unchanged addons are staged from cache."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
@@ -78,6 +86,14 @@ def buildable_addons(root: Path) -> list[str]:
|
||||
return addons
|
||||
|
||||
|
||||
def parse_addon_allowlist(value: str | None) -> list[str] | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
addons = [part.strip() for part in value.split(",") if part.strip()]
|
||||
return sorted(dict.fromkeys(addons))
|
||||
|
||||
|
||||
def file_digest(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
@@ -289,12 +305,24 @@ def main() -> int:
|
||||
stage_dir = (root / args.stage_dir).resolve()
|
||||
|
||||
addons = buildable_addons(root)
|
||||
requested_addons = parse_addon_allowlist(args.addons)
|
||||
if requested_addons is not None:
|
||||
unknown_addons = sorted(set(requested_addons) - set(addons))
|
||||
if unknown_addons:
|
||||
print(
|
||||
"ERROR: Requested addon is not buildable: "
|
||||
+ ", ".join(unknown_addons),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
if args.source_root:
|
||||
source_root = Path(args.source_root).resolve()
|
||||
elif args.dry_run:
|
||||
source_root = (cache_dir / "mikero-source" / "braf").resolve()
|
||||
else:
|
||||
source_root = prepare_mikero_source_root(root, cache_dir, addons).resolve()
|
||||
source_root_addons = requested_addons if requested_addons is not None else addons
|
||||
source_root = prepare_mikero_source_root(root, cache_dir, source_root_addons).resolve()
|
||||
|
||||
current_hashes = {addon: addon_digest(root / addon) for addon in addons}
|
||||
manifest = load_manifest(manifest_path)
|
||||
@@ -308,7 +336,8 @@ def main() -> int:
|
||||
manifest_addons.pop(addon, None)
|
||||
|
||||
to_build: list[str] = []
|
||||
for addon in addons:
|
||||
candidate_addons = requested_addons if requested_addons is not None else addons
|
||||
for addon in candidate_addons:
|
||||
cached_hash = manifest_addons.get(addon, {}).get("hash")
|
||||
if (
|
||||
args.force_rebuild
|
||||
@@ -318,6 +347,11 @@ def main() -> int:
|
||||
to_build.append(addon)
|
||||
|
||||
print(f"Buildable addons: {len(addons)}")
|
||||
if requested_addons is not None:
|
||||
print(
|
||||
"Requested rebuild allowlist: "
|
||||
+ (", ".join(requested_addons) if requested_addons else "none")
|
||||
)
|
||||
print(f"Addons to build: {', '.join(to_build) if to_build else 'none'}")
|
||||
print(f"Cache dir: {cache_dir}")
|
||||
print(f"Stage dir: {stage_dir}")
|
||||
@@ -336,7 +370,15 @@ def main() -> int:
|
||||
|
||||
for addon in addons:
|
||||
if not cached_pbo_exists(cache_addons_dir, addon):
|
||||
raise RuntimeError(f"Cache does not contain required PBO after build: {addon}")
|
||||
hint = (
|
||||
" Seed the persistent cache first or include this addon in the "
|
||||
"detected rebuild set."
|
||||
if requested_addons is not None and addon not in candidate_addons
|
||||
else ""
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"Cache does not contain required PBO after build: {addon}.{hint}"
|
||||
)
|
||||
|
||||
stage_package(root, stage_dir, cache_addons_dir, addons)
|
||||
write_manifest(manifest_path, manifest)
|
||||
|
||||
Reference in New Issue
Block a user