feat: add manifest-driven batch rebuild, numpy cross-compilation, and CI pipeline updates

- Add pipeline/manifest.py for packages.txt parsing
- Implement real numpy cross-compilation (rebuild.py) with OHOS NDK + vendored meson
- Extend CLI with --manifest, --build-from-source, --stage-integration modes
- Add recipes/, experiments/, and scripts/ for integration staging and artifact upload
- Update CI workflow with manifest validation, artifact upload, and release publishing
- Update Makefile and README with new targets and documentation

pipeline/cli.py

@@ -1,22 +1,131 @@
 import argparse
+import glob
 import os
+import subprocess
 import sys
+import zipfile
 
 from . import detect, repack, rebuild
+from .manifest import parse_manifest
+
+# Lazy import for integration staging (optional dependency)
+try:
+    from scripts.stage_integration import stage_integration as _stage_integration
+except ImportError:
+    _stage_integration = None
+
+BUILD_RECIPES = {
+    "numpy": rebuild.build_numpy_wheel,
+}
+
+MANIFEST_SOURCES_DIR = "output/sources"
+
+
+def _clone_repo(repo_url, pkg_name):
+    dest = os.path.join(MANIFEST_SOURCES_DIR, pkg_name)
+    if os.path.isdir(dest) and os.listdir(dest):
+        print(f"[manifest] Source already exists: {dest}")
+        return dest
+    os.makedirs(dest, exist_ok=True)
+    try:
+        print(f"[manifest] Cloning {repo_url} \u2192 {dest}")
+        subprocess.run(
+            ["git", "clone", "--depth", "1", repo_url, dest],
+            check=True, capture_output=True, timeout=120,
+        )
+    except Exception as e:
+        print(f"[manifest] Warning: git clone failed for {pkg_name}: {e}")
+    return dest
+
+
+def _collect_wheels(wheel_dir, output_dir, zip_name="ecoark-rebuild-output.zip"):
+    whls = sorted(glob.glob(os.path.join(wheel_dir, "*.whl")))
+    zip_path = os.path.join(output_dir, zip_name)
+    os.makedirs(os.path.dirname(zip_path), exist_ok=True)
+    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+        for whl in whls:
+            zf.write(whl, os.path.basename(whl))
+    print(f"[manifest] Bundle: {zip_path} ({len(whls)} wheel(s))")
+    return zip_path
+
+
+def run_manifest(manifest_path, output_dir, target_os="harmonyos", arch="arm64"):
+    os.makedirs(output_dir, exist_ok=True)
+    entries = parse_manifest(manifest_path)
+    if not entries:
+        print(f"[manifest] No active entries in {manifest_path} \u2014 nothing to build.")
+        print("[manifest] Edit packages.txt or create a temporary manifest to proceed.")
+        _collect_wheels(output_dir, output_dir)
+        return
+
+    print(f"[manifest] Processing {len(entries)} package(s) from {manifest_path}")
+    wheel_dir = os.path.join(output_dir, "wheels")
+
+    for pkg_name, repo_url in entries:
+        print(f"\n{'=' * 60}")
+        print(f"[manifest] Package: {pkg_name}")
+        print(f"[manifest] Repo   : {repo_url}")
+        src = _clone_repo(repo_url, pkg_name)
+        rebuild.rebuild_from_source(src, pkg_name, wheel_dir, target_os=target_os, arch=arch)
+
+    bundle = _collect_wheels(wheel_dir, output_dir)
+    print(f"\n{'=' * 60}")
+    print(f"[manifest] Done. Bundle: {bundle}")
 
 
 def main():
     parser = argparse.ArgumentParser(
-        description="EcoArk HarmonyOS Wheel Pipeline — repack / rebuild Python wheels for HarmonyOS/arm64"
+        description="EcoArk HarmonyOS Wheel Pipeline \u2014 repack / rebuild Python wheels for HarmonyOS/arm64"
     )
-    parser.add_argument("input", help="Path to a .whl file")
+    parser.add_argument("input", nargs="?", help="Path to a .whl file (single-wheel mode)")
+    parser.add_argument("--manifest", help="Path to packages manifest file (manifest-driven mode)")
+    parser.add_argument("--build-from-source", choices=list(BUILD_RECIPES.keys()),
+                        help="Build a package from source (cross-compile for HarmonyOS)")
+    parser.add_argument("--package-version", default="1.26.4",
+                        help="Package version for --build-from-source (default: 1.26.4)")
     parser.add_argument("--output-dir", "-o", default="output", help="Output directory (default: ./output)")
     parser.add_argument("--target-os", default="harmonyos", choices=["harmonyos", "linux", "ohos"])
     parser.add_argument("--arch", default="arm64", choices=["arm64", "aarch64", "x86_64"])
     parser.add_argument("--force-native", action="store_true", help="Treat as native wheel even if detection says pure")
     parser.add_argument("--force-repack", action="store_true", help="Force repack even for native wheels")
+    parser.add_argument("--rebuild", action="store_true", help="Force rebuild even if build dir exists")
+    parser.add_argument("--stage-integration", nargs="?",
+                        const="output/wheels/numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl",
+                        help="Stage native .so files for EcoArk integration (optional: specify wheel path)")
     args = parser.parse_args()
 
+    if args.build_from_source:
+        builder = BUILD_RECIPES[args.build_from_source]
+        output_dir = os.path.abspath(args.output_dir)
+        os.makedirs(output_dir, exist_ok=True)
+        wheel_path = builder(
+            version=args.package_version,
+            output_dir=output_dir,
+            rebuild=args.rebuild,
+        )
+        if wheel_path:
+            print(f"[pipeline] Done. Wheel: {wheel_path}")
+        else:
+            print("[pipeline] FAILED", file=sys.stderr)
+            sys.exit(1)
+        return
+
+    if args.manifest:
+        run_manifest(args.manifest, args.output_dir, args.target_os, args.arch)
+        return
+
+    if args.stage_integration:
+        if _stage_integration is None:
+            print("[pipeline] ERROR: scripts.stage_integration not importable", file=sys.stderr)
+            sys.exit(1)
+        wheel = os.path.abspath(args.stage_integration)
+        out = os.path.join(os.path.abspath(args.output_dir), "integration")
+        rc = _stage_integration(wheel, out)
+        sys.exit(rc)
+
+    if not args.input:
+        parser.error("either INPUT (a .whl file), --manifest, --build-from-source, or --stage-integration is required")
+
     input_path = args.input
     if not os.path.isfile(input_path):
         print(f"Error: input file not found: {input_path}", file=sys.stderr)
@@ -40,23 +149,23 @@ def main():
     is_native = info["native"] or args.force_native
 
     if not is_native:
-        print("[pipeline] Pure Python wheel — repack with harmonyos tag")
+        print("[pipeline] Pure Python wheel \u2014 repack with harmonyos tag")
         out = repack.repack_wheel(input_path, output_dir)
         print(f"[pipeline] Output   : {out}")
-        print("[pipeline] Done — wheel can be installed directly on HarmonyOS.")
+        print("[pipeline] Done \u2014 wheel can be installed directly on HarmonyOS.")
     else:
         if args.force_repack:
             print("[pipeline] Pure wheel detected but force-repack requested.")
             out = repack.repack_wheel(input_path, output_dir)
             print(f"[pipeline] Repacked  : {out}")
         else:
-            print("[pipeline] Native wheel — needs cross-compilation.")
+            print("[pipeline] Native wheel \u2014 needs cross-compilation.")
             out = rebuild.rebuild_native_wheel(
                 input_path, output_dir,
                 target_os=args.target_os, arch=args.arch,
             )
             print(f"[pipeline] Output     : {out}")
-            print("[pipeline] Done — native rebuild entry point triggered.")
+            print("[pipeline] Done \u2014 native rebuild entry point triggered.")
 
 
 if __name__ == "__main__":