Valmo Trindade
541 lines
17 KiB
Python
541 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Build and stage the BRAF DEV Steam Workshop package.
|
|
|
|
The script keeps a persistent addon cache outside the checkout. Each run hashes
|
|
root-level braf_* addon folders, rebuilds only changed or missing PBOs with
|
|
Mikero pboProject, then stages a complete Arma 3 mod folder for upload.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from typing import Any
|
|
|
|
try:
|
|
from make.obsfuschatedList import obfuschatedList
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from make.obsfuschatedList import obfuschatedList
|
|
|
|
|
|
MANIFEST_VERSION = 1
|
|
DEFAULT_STAGE_DIR = Path("dist/dev/@braf_dev")
|
|
METADATA_FILES = ("mod.cpp", "meta.cpp")
|
|
MOD_ASSET_FIELDS = ("picture", "logo", "logoOver", "logoSmall")
|
|
IGNORED_DIRS = {".git", "__pycache__"}
|
|
CASE_ALIAS_EXTENSIONS = (
|
|
"bisurf",
|
|
"csv",
|
|
"fsm",
|
|
"hpp",
|
|
"jpg",
|
|
"ogg",
|
|
"paa",
|
|
"p3d",
|
|
"png",
|
|
"rvmat",
|
|
"rtm",
|
|
"sqf",
|
|
"txt",
|
|
"wss",
|
|
"xml",
|
|
)
|
|
CASE_ALIAS_RE = re.compile(
|
|
rb"[A-Za-z0-9_ ./\\-]+\.("
|
|
+ b"|".join(ext.encode("ascii") for ext in CASE_ALIAS_EXTENSIONS)
|
|
+ rb")",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Build changed BRAF addons and stage the DEV workshop folder."
|
|
)
|
|
parser.add_argument(
|
|
"--cache-dir",
|
|
default=os.getenv("BRAF_CI_CACHE_DIR"),
|
|
help="Persistent cache directory. Defaults to BRAF_CI_CACHE_DIR.",
|
|
)
|
|
parser.add_argument(
|
|
"--stage-dir",
|
|
default=os.getenv("BRAF_CI_STAGE_DIR", str(DEFAULT_STAGE_DIR)),
|
|
help=f"Output mod folder. Defaults to {DEFAULT_STAGE_DIR}.",
|
|
)
|
|
parser.add_argument(
|
|
"--source-root",
|
|
default=os.getenv("BRAF_MIKERO_SOURCE_ROOT"),
|
|
help="Root passed to pboProject. Defaults to the repository checkout.",
|
|
)
|
|
parser.add_argument(
|
|
"--force-rebuild",
|
|
action="store_true",
|
|
help="Rebuild every buildable addon, ignoring matching cache hashes.",
|
|
)
|
|
parser.add_argument(
|
|
"--addons",
|
|
default=None,
|
|
help=(
|
|
"Comma-separated addon allowlist to rebuild. When set, only these "
|
|
"addons may be rebuilt; unchanged addons are staged from cache."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="Print planned work without running pboProject or writing cache/stage output.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def repo_root() -> Path:
|
|
return Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def buildable_addons(root: Path) -> list[str]:
|
|
addons: list[str] = []
|
|
for path in sorted(root.glob("braf_*")):
|
|
if path.is_dir() and (path / "config.cpp").is_file():
|
|
addons.append(path.name)
|
|
return addons
|
|
|
|
|
|
def parse_addon_allowlist(value: str | None) -> list[str] | None:
|
|
if value is None:
|
|
return None
|
|
|
|
addons = [part.strip() for part in value.split(",") if part.strip()]
|
|
return sorted(dict.fromkeys(addons))
|
|
|
|
|
|
def file_digest(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def addon_digest(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
for file_path in sorted(p for p in path.rglob("*") if p.is_file()):
|
|
if any(part in IGNORED_DIRS for part in file_path.relative_to(path).parts):
|
|
continue
|
|
rel = file_path.relative_to(path).as_posix()
|
|
digest.update(rel.encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(file_digest(file_path).encode("ascii"))
|
|
digest.update(b"\0")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def load_manifest(path: Path) -> dict[str, Any]:
|
|
if not path.is_file():
|
|
return {"version": MANIFEST_VERSION, "addons": {}}
|
|
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
data = json.load(handle)
|
|
|
|
if data.get("version") != MANIFEST_VERSION or not isinstance(data.get("addons"), dict):
|
|
return {"version": MANIFEST_VERSION, "addons": {}}
|
|
|
|
return data
|
|
|
|
|
|
def write_manifest(path: Path, manifest: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(".tmp")
|
|
with tmp.open("w", encoding="utf-8") as handle:
|
|
json.dump(manifest, handle, indent=2, sort_keys=True)
|
|
handle.write("\n")
|
|
tmp.replace(path)
|
|
|
|
|
|
def cached_pbo_exists(cache_addons_dir: Path, addon: str) -> bool:
|
|
return (cache_addons_dir / f"{addon}.pbo").is_file()
|
|
|
|
|
|
def addon_cache_files(cache_addons_dir: Path, addon: str) -> list[Path]:
|
|
patterns = (f"{addon}.pbo", f"{addon}.pbo.*.bisign")
|
|
files: list[Path] = []
|
|
for pattern in patterns:
|
|
files.extend(sorted(cache_addons_dir.glob(pattern)))
|
|
return files
|
|
|
|
|
|
def remove_cached_addon(cache_addons_dir: Path, addon: str) -> None:
|
|
for path in addon_cache_files(cache_addons_dir, addon):
|
|
path.unlink()
|
|
|
|
|
|
def rel_symlink(src: Path, dst: Path) -> None:
|
|
target = os.path.relpath(src, start=dst.parent)
|
|
dst.symlink_to(target, target_is_directory=src.is_dir())
|
|
|
|
|
|
def create_source_symlink_tree(src: Path, dst: Path) -> dict[str, Path]:
|
|
lower_to_source: dict[str, Path] = {}
|
|
for path in sorted(src.rglob("*")):
|
|
rel = path.relative_to(src)
|
|
target = dst / rel
|
|
if path.is_dir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
continue
|
|
if any(part in IGNORED_DIRS for part in rel.parts):
|
|
continue
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
if target.exists() or target.is_symlink():
|
|
target.unlink()
|
|
rel_symlink(path, target)
|
|
lower_to_source[rel.as_posix().lower()] = path
|
|
return lower_to_source
|
|
|
|
|
|
def extract_case_alias_refs(addon: str, file_path: Path) -> set[str]:
|
|
refs: set[str] = set()
|
|
try:
|
|
data = file_path.read_bytes()
|
|
except OSError:
|
|
return refs
|
|
|
|
addon_prefixes = (
|
|
f"braf/{addon}/",
|
|
f"{addon}/",
|
|
)
|
|
for match in CASE_ALIAS_RE.finditer(data):
|
|
value = match.group(0).decode("latin-1", errors="ignore")
|
|
value = value.strip(" \t\r\n\"'()[]{};,")
|
|
value = value.replace("\\", "/").lstrip("/")
|
|
lower_value = value.lower()
|
|
|
|
rel = None
|
|
for prefix in addon_prefixes:
|
|
index = lower_value.find(prefix)
|
|
if index != -1:
|
|
rel = value[index + len(prefix) :]
|
|
break
|
|
if rel is None and "/" in value:
|
|
rel = value
|
|
elif rel is None:
|
|
continue
|
|
|
|
rel_path = Path(rel)
|
|
if rel_path.is_absolute() or ".." in rel_path.parts:
|
|
continue
|
|
refs.add(rel_path.as_posix())
|
|
return refs
|
|
|
|
|
|
def add_case_aliases(addon: str, src: Path, dst: Path, lower_to_source: dict[str, Path]) -> None:
|
|
wanted_refs = {rel.lower(): rel for rel in lower_to_source}
|
|
for file_path in sorted(path for path in src.rglob("*") if path.is_file()):
|
|
for ref in extract_case_alias_refs(addon, file_path):
|
|
wanted_refs.setdefault(ref.lower(), ref)
|
|
|
|
for lower_ref, requested_ref in sorted(wanted_refs.items()):
|
|
source = lower_to_source.get(lower_ref)
|
|
if source is None:
|
|
continue
|
|
alias = dst / requested_ref
|
|
if alias.exists() or alias.is_symlink():
|
|
continue
|
|
alias.parent.mkdir(parents=True, exist_ok=True)
|
|
rel_symlink(source, alias)
|
|
|
|
|
|
def raw_pbo_files(source: Path) -> list[tuple[str, Path, int]]:
|
|
files: list[tuple[str, Path, int]] = []
|
|
for file_path in sorted(
|
|
(path for path in source.rglob("*") if path.is_file()),
|
|
key=lambda path: path.relative_to(source).as_posix().lower(),
|
|
):
|
|
rel = file_path.relative_to(source)
|
|
if any(part in IGNORED_DIRS for part in rel.parts):
|
|
continue
|
|
files.append((rel.as_posix().replace("/", "\\"), file_path, file_path.stat().st_size))
|
|
return files
|
|
|
|
|
|
def write_raw_pbo(addon: str, source: Path, output_path: Path) -> None:
|
|
files = raw_pbo_files(source)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
timestamp = int(time.time())
|
|
sha = hashlib.sha1()
|
|
|
|
with output_path.open("wb") as handle:
|
|
def write(data: bytes) -> None:
|
|
handle.write(data)
|
|
sha.update(data)
|
|
|
|
write(b"\0")
|
|
write(struct.pack("<IIIII", 0x56657273, 0, 0, 0, 0))
|
|
for key, value in (
|
|
("prefix", f"braf\\{addon}"),
|
|
("version", "raw-dev-ci"),
|
|
):
|
|
write(key.encode("utf-8") + b"\0")
|
|
write(value.encode("utf-8") + b"\0")
|
|
write(b"\0")
|
|
|
|
for rel, _file_path, size in files:
|
|
write(rel.encode("utf-8") + b"\0")
|
|
write(struct.pack("<IIIII", 0, size, 0, timestamp, size))
|
|
|
|
write(b"\0")
|
|
write(struct.pack("<IIIII", 0, 0, 0, 0, 0))
|
|
|
|
for _rel, file_path, _size in files:
|
|
with file_path.open("rb") as input_file:
|
|
for chunk in iter(lambda: input_file.read(1024 * 1024), b""):
|
|
write(chunk)
|
|
|
|
handle.write(sha.digest())
|
|
|
|
|
|
def prepare_mikero_source_root(root: Path, cache_dir: Path, addons: list[str]) -> Path:
|
|
"""Create a P:/braf-like source root with case aliases for Linux Mikero.
|
|
|
|
The historic local build path was P:/braf/<addon>. Keeping the parent folder
|
|
named "braf" avoids changing Mikero's inferred addon prefix in CI.
|
|
"""
|
|
source_root = cache_dir / "mikero-source" / "braf"
|
|
source_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
wanted = set(addons)
|
|
for child in source_root.iterdir():
|
|
if child.name not in wanted:
|
|
if child.is_symlink() or child.is_file():
|
|
child.unlink()
|
|
elif child.is_dir():
|
|
shutil.rmtree(child)
|
|
|
|
for addon in addons:
|
|
dst = source_root / addon
|
|
src = root / addon
|
|
if dst.exists() or dst.is_symlink():
|
|
if dst.is_dir() and not dst.is_symlink():
|
|
shutil.rmtree(dst)
|
|
else:
|
|
dst.unlink()
|
|
dst.mkdir(parents=True)
|
|
lower_to_source = create_source_symlink_tree(src, dst)
|
|
add_case_aliases(addon, src, dst, lower_to_source)
|
|
|
|
return source_root
|
|
|
|
|
|
def run_pboproject(addon: str, source_root: Path, output_dir: Path) -> None:
|
|
addon_source = source_root / addon
|
|
if not addon_source.is_dir():
|
|
raise RuntimeError(f"Mikero source directory not found: {addon_source}")
|
|
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
pboproject = shutil.which("pboproject") or shutil.which("pboProject")
|
|
|
|
if pboproject:
|
|
cmd = [
|
|
pboproject,
|
|
"-P",
|
|
str(addon_source),
|
|
f"+M={output_dir}",
|
|
"+O" if addon in obfuschatedList else "-O",
|
|
]
|
|
else:
|
|
source = repo_root() / addon
|
|
fallback = output_dir / f"{addon}.pbo"
|
|
print(
|
|
"Mikero pboProject is not available; writing raw DEV PBO "
|
|
f"for {addon} at {fallback}"
|
|
)
|
|
write_raw_pbo(addon, source, fallback)
|
|
return
|
|
|
|
print(f"::group::Building {addon}")
|
|
print(" ".join(cmd))
|
|
try:
|
|
subprocess.run(cmd, check=True)
|
|
finally:
|
|
print("::endgroup::")
|
|
|
|
|
|
def promote_build_output(addon: str, build_dir: Path, cache_addons_dir: Path) -> None:
|
|
pbo = build_dir / f"{addon}.pbo"
|
|
if not pbo.is_file():
|
|
candidates = sorted(build_dir.glob("*.pbo"))
|
|
candidate_names = ", ".join(path.name for path in candidates) or "none"
|
|
raise RuntimeError(f"Expected {pbo.name} from pboProject; found: {candidate_names}")
|
|
|
|
cache_addons_dir.mkdir(parents=True, exist_ok=True)
|
|
remove_cached_addon(cache_addons_dir, addon)
|
|
|
|
for path in sorted(build_dir.glob(f"{addon}.pbo*")):
|
|
if path.is_file():
|
|
shutil.copy2(path, cache_addons_dir / path.name)
|
|
|
|
|
|
def parse_mod_assets(mod_cpp: Path) -> list[Path]:
|
|
if not mod_cpp.is_file():
|
|
return []
|
|
|
|
text = mod_cpp.read_text(encoding="utf-8", errors="ignore")
|
|
assets: list[Path] = []
|
|
for field in MOD_ASSET_FIELDS:
|
|
pattern = re.compile(rf"\b{re.escape(field)}\s*=\s*\"([^\"]*)\"", re.IGNORECASE)
|
|
for match in pattern.finditer(text):
|
|
value = match.group(1).strip()
|
|
if value:
|
|
assets.append(Path(value.replace("\\", "/")))
|
|
return assets
|
|
|
|
|
|
def copy_metadata(root: Path, stage_dir: Path) -> None:
|
|
for name in METADATA_FILES:
|
|
src = root / name
|
|
if not src.is_file():
|
|
raise RuntimeError(f"Required metadata file is missing: {name}")
|
|
shutil.copy2(src, stage_dir / name)
|
|
|
|
for rel_asset in parse_mod_assets(root / "mod.cpp"):
|
|
if rel_asset.is_absolute() or ".." in rel_asset.parts:
|
|
raise RuntimeError(f"Unsupported mod.cpp asset path: {rel_asset}")
|
|
src = root / rel_asset
|
|
if not src.is_file():
|
|
raise RuntimeError(f"mod.cpp references missing asset: {rel_asset}")
|
|
dst = stage_dir / rel_asset
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, dst)
|
|
|
|
|
|
def stage_package(
|
|
root: Path,
|
|
stage_dir: Path,
|
|
cache_addons_dir: Path,
|
|
addons: list[str],
|
|
) -> None:
|
|
if stage_dir.exists():
|
|
shutil.rmtree(stage_dir)
|
|
addons_dir = stage_dir / "addons"
|
|
addons_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
copy_metadata(root, stage_dir)
|
|
|
|
for addon in addons:
|
|
files = addon_cache_files(cache_addons_dir, addon)
|
|
if not any(path.name == f"{addon}.pbo" for path in files):
|
|
raise RuntimeError(f"Missing cached PBO for {addon}")
|
|
for path in files:
|
|
shutil.copy2(path, addons_dir / path.name)
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
root = repo_root()
|
|
|
|
if not args.cache_dir:
|
|
print("ERROR: --cache-dir or BRAF_CI_CACHE_DIR is required", file=sys.stderr)
|
|
return 2
|
|
|
|
cache_dir = Path(args.cache_dir).resolve()
|
|
cache_addons_dir = cache_dir / "addons"
|
|
manifest_path = cache_dir / "manifest.json"
|
|
stage_dir = (root / args.stage_dir).resolve()
|
|
|
|
addons = buildable_addons(root)
|
|
requested_addons = parse_addon_allowlist(args.addons)
|
|
if requested_addons is not None:
|
|
unknown_addons = sorted(set(requested_addons) - set(addons))
|
|
if unknown_addons:
|
|
print(
|
|
"ERROR: Requested addon is not buildable: "
|
|
+ ", ".join(unknown_addons),
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
if args.source_root:
|
|
source_root = Path(args.source_root).resolve()
|
|
elif args.dry_run:
|
|
source_root = (cache_dir / "mikero-source" / "braf").resolve()
|
|
elif not (shutil.which("pboproject") or shutil.which("pboProject")):
|
|
source_root = root
|
|
else:
|
|
source_root_addons = requested_addons if requested_addons is not None else addons
|
|
source_root = prepare_mikero_source_root(root, cache_dir, source_root_addons).resolve()
|
|
|
|
current_hashes = {addon: addon_digest(root / addon) for addon in addons}
|
|
manifest = load_manifest(manifest_path)
|
|
manifest_addons = manifest.setdefault("addons", {})
|
|
|
|
removed_addons = sorted(set(manifest_addons) - set(addons))
|
|
for addon in removed_addons:
|
|
print(f"Removing stale addon from cache manifest: {addon}")
|
|
if not args.dry_run:
|
|
remove_cached_addon(cache_addons_dir, addon)
|
|
manifest_addons.pop(addon, None)
|
|
|
|
to_build: list[str] = []
|
|
candidate_addons = requested_addons if requested_addons is not None else addons
|
|
for addon in candidate_addons:
|
|
cached_hash = manifest_addons.get(addon, {}).get("hash")
|
|
if (
|
|
args.force_rebuild
|
|
or cached_hash != current_hashes[addon]
|
|
or not cached_pbo_exists(cache_addons_dir, addon)
|
|
):
|
|
to_build.append(addon)
|
|
|
|
print(f"Buildable addons: {len(addons)}")
|
|
if requested_addons is not None:
|
|
print(
|
|
"Requested rebuild allowlist: "
|
|
+ (", ".join(requested_addons) if requested_addons else "none")
|
|
)
|
|
print(f"Addons to build: {', '.join(to_build) if to_build else 'none'}")
|
|
print(f"Cache dir: {cache_dir}")
|
|
print(f"Stage dir: {stage_dir}")
|
|
print(f"Mikero source root: {source_root}")
|
|
|
|
if args.dry_run:
|
|
return 0
|
|
|
|
for addon in to_build:
|
|
with TemporaryDirectory(prefix=f"{addon}-", dir=str(cache_dir)) as tmp:
|
|
build_dir = Path(tmp)
|
|
run_pboproject(addon, source_root, build_dir)
|
|
promote_build_output(addon, build_dir, cache_addons_dir)
|
|
manifest_addons[addon] = {"hash": current_hashes[addon]}
|
|
write_manifest(manifest_path, manifest)
|
|
|
|
for addon in addons:
|
|
if not cached_pbo_exists(cache_addons_dir, addon):
|
|
hint = (
|
|
" Seed the persistent cache first or include this addon in the "
|
|
"detected rebuild set."
|
|
if requested_addons is not None and addon not in candidate_addons
|
|
else ""
|
|
)
|
|
raise RuntimeError(
|
|
f"Cache does not contain required PBO after build: {addon}.{hint}"
|
|
)
|
|
|
|
stage_package(root, stage_dir, cache_addons_dir, addons)
|
|
write_manifest(manifest_path, manifest)
|
|
|
|
print(f"Staged {len(addons)} addons at {stage_dir}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|