feat: add manifest-driven batch rebuild, numpy cross-compilation, and CI pipeline updates

- Add pipeline/manifest.py for packages.txt parsing
- Implement real numpy cross-compilation (rebuild.py) with OHOS NDK + vendored meson
- Extend CLI with --manifest, --build-from-source, --stage-integration modes
- Add recipes/, experiments/, and scripts/ for integration staging and artifact upload
- Update CI workflow with manifest validation, artifact upload, and release publishing
- Update Makefile and README with new targets and documentation

scripts/stage_integration.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Stage built native .so files for EcoArk pythonrunner main-so integration.
+
+Extracts .so files from an existing HarmonyOS-cross-compiled wheel and copies
+them into an integration output directory with a README describing where each
+module should be placed in the EcoArk pythonrunner environment.
+"""
+import argparse
+import json
+import os
+import shutil
+import sys
+import zipfile
+
+
+def stage_integration(wheel_path, output_dir, recipe=None):
+    os.makedirs(output_dir, exist_ok=True)
+
+    if not os.path.isfile(wheel_path):
+        print(f"[stage] ERROR: wheel not found: {wheel_path}", file=sys.stderr)
+        return 1
+
+    staged = []
+    with zipfile.ZipFile(wheel_path, "r") as z:
+        for name in z.namelist():
+            if not name.endswith(".so"):
+                continue
+            dest = os.path.join(output_dir, name)
+            os.makedirs(os.path.dirname(dest), exist_ok=True)
+            with z.open(name) as src, open(dest, "wb") as dst:
+                shutil.copyfileobj(src, dst)
+            staged.append(name)
+            print(f"[stage]  {name}")
+
+    readme_path = os.path.join(output_dir, "README.md")
+    with open(readme_path, "w") as f:
+        f.write(f"# Integration Staging for EcoArk pythonrunner main-so linking\n\n")
+        f.write(f"Source wheel: `{wheel_path}`\n\n")
+        f.write(f"## Staged .so files ({len(staged)} total)\n\n")
+        for s in staged:
+            f.write(f"- `{s}`\n")
+        f.write(f"\n## EcoArk target hint\n\n")
+        f.write("Copy each .so into EcoArk pythonrunner's native module tree\n")
+        f.write("under `lib/python3.12/site-packages/` preserving subdirectory\n")
+        f.write("structure so that `import numpy` resolves via main-so linking.\n")
+        so_list_path = os.path.join(output_dir, "integration_manifest.json")
+        manifest = {
+            "pkg": "numpy",
+            "staged_count": len(staged),
+            "ecoark_target": "lib/python3.12/site-packages/",
+            "files": staged,
+        }
+        if recipe and os.path.isfile(recipe):
+            with open(recipe) as rf:
+                manifest["recipe"] = json.load(rf)
+    with open(so_list_path, "w") as f:
+        json.dump(manifest, f, indent=2)
+
+    print(f"\n[stage] Staged {len(staged)} .so files to {output_dir}")
+    print(f"[stage] Manifest: {so_list_path}")
+    print(f"[stage] README:   {readme_path}")
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Stage native .so files from a built HarmonyOS wheel for EcoArk integration"
+    )
+    parser.add_argument("wheel", nargs="?", help="Path to built HarmonyOS .whl file")
+    parser.add_argument("--output-dir", "-o", default="output/integration",
+                        help="Integration output directory (default: output/integration)")
+    parser.add_argument("--recipe", help="Path to integration recipe JSON (optional)")
+    parser.add_argument("--list-recipes", action="store_true",
+                        help="List available recipes and exit")
+    args = parser.parse_args()
+
+    if args.list_recipes:
+        recipes_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "recipes")
+        print("Available integration recipes:")
+        if os.path.isdir(recipes_dir):
+            for fn in sorted(os.listdir(recipes_dir)):
+                if fn.endswith(".json"):
+                    rp = os.path.join(recipes_dir, fn)
+                    with open(rp) as f:
+                        d = json.load(f)
+                    print(f"  {fn}  ({d.get('pkg','?')} {d.get('version','?')})")
+        else:
+            print("  (no recipes directory)")
+        return 0
+
+    if not args.wheel:
+        # Default: look for numpy wheel in output/wheels
+        default = os.path.join(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+            "output", "wheels", "numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl"
+        )
+        if os.path.isfile(default):
+            args.wheel = default
+            print(f"[stage] Using default wheel: {default}")
+        else:
+            parser.error("no wheel specified and default numpy wheel not found")
+
+    return stage_integration(args.wheel, os.path.abspath(args.output_dir), recipe=args.recipe)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/upload_artifacts.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""
+Upload build outputs as Gitea Actions artifacts using the Actions Runtime API.
+
+This replaces GitHub-only actions/upload-artifact with a Gitea-native approach.
+Uses ACTIONS_RUNTIME_URL and ACTIONS_RUNTIME_TOKEN provided by Gitea's
+act_runner to upload artifacts directly to the pipeline service.
+
+Usage:
+  python3 scripts/upload_artifacts.py <artifact_name> <path_glob1> [path_glob2 ...]
+
+Environment (set by Gitea act_runner >= 1.21):
+  ACTIONS_RUNTIME_URL     — Pipeline service base URL (e.g. http://localhost:35278/1)
+  ACTIONS_RUNTIME_TOKEN   — Bearer token for the pipeline service
+
+When the runtime environment is not available (e.g. local testing),
+artifacts are saved to ./artifacts/<name>.zip as a fallback.
+"""
+import glob
+import io
+import os
+import sys
+import urllib.error
+import urllib.request
+import zipfile
+
+_RUNTIME_UPLOADED = False
+
+
+def _upload_via_runtime(name, payload, runtime_url, runtime_token):
+    url = "{}/_apis/pipeline/artifacts/{}".format(runtime_url.rstrip("/"), name)
+    headers = {
+        "Authorization": "Bearer " + runtime_token,
+        "Content-Type": "application/octet-stream",
+        "Content-Range": "bytes 0-{}/{}".format(len(payload) - 1, len(payload)),
+        "x-tfs-filelength": str(len(payload)),
+    }
+    req = urllib.request.Request(url, data=payload, headers=headers, method="PUT")
+    try:
+        with urllib.request.urlopen(req) as resp:
+            msg = "[upload-artifact] Uploaded '{}' ({} bytes) to Gitea runtime — HTTP {}"
+            print(msg.format(name, len(payload), resp.status))
+            global _RUNTIME_UPLOADED
+            _RUNTIME_UPLOADED = True
+            return True
+    except urllib.error.HTTPError as e:
+        body = e.read().decode()[:300] if e.fp else ""
+        print("[upload-artifact] Runtime upload failed: HTTP {} — {}".format(e.code, body))
+        return False
+
+
+def _save_locally(name, payload):
+    dest = os.path.join("artifacts", "{}.zip".format(name))
+    os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
+    with open(dest, "wb") as f:
+        f.write(payload)
+    print("[upload-artifact] Saved artifact to {} ({} bytes)".format(dest, len(payload)))
+
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: {} <artifact_name> <path_glob> [path_glob ...]".format(sys.argv[0]))
+        sys.exit(1)
+
+    artifact_name = sys.argv[1]
+    patterns = sys.argv[2:]
+
+    files = []
+    for pattern in patterns:
+        matched = glob.glob(pattern, recursive=True)
+        files.extend(matched)
+    files = sorted(set(f for f in files if os.path.isfile(f)))
+
+    if not files:
+        print("[upload-artifact] No files found for pattern(s): {}".format(patterns))
+        return
+
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        for f in files:
+            zf.write(f, os.path.relpath(f))
+    payload = buf.getvalue()
+
+    runtime_url = os.environ.get("ACTIONS_RUNTIME_URL", "").rstrip("/")
+    runtime_token = os.environ.get("ACTIONS_RUNTIME_TOKEN", "")
+
+    if runtime_url and runtime_token:
+        ok = _upload_via_runtime(artifact_name, payload, runtime_url, runtime_token)
+        if not ok:
+            _save_locally(artifact_name, payload)
+    else:
+        print("[upload-artifact] ACTIONS_RUNTIME_URL/TOKEN not available — saving locally.")
+        _save_locally(artifact_name, payload)
+
+    print("[upload-artifact] Done — {} file(s) in '{}'.".format(len(files), artifact_name))
+
+
+if __name__ == "__main__":
+    main()

scripts/upload_release_assets.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+Upload wheel / zip assets to a Gitea Release.
+
+Environment variables (set by Gitea runner or user):
+  ENABLE_RELEASE_UPLOAD  — set to "true" to activate (default: skip)
+  GITEA_TOKEN            — Gitea access token with `repo` scope
+  RELEASE_TOKEN          — fallback if GITEA_TOKEN is unset
+  GITHUB_SERVER_URL      — Gitea server URL (runner-provided)
+  GITHUB_REPOSITORY      — owner/repo (runner-provided)
+  GITHUB_REF_NAME        — tag name when pushing a tag (runner-provided)
+  GITHUB_SHA             — commit SHA (runner-provided)
+
+Usage:
+  python3 scripts/upload_release_assets.py <assets-dir>
+"""
+import glob
+import json
+import os
+import secrets
+import sys
+import urllib.error
+import urllib.request
+
+
+def env(key, default=""):
+    return os.environ.get(key, default)
+
+
+def _multipart_formdata(field, filename, filepath):
+    boundary = "----GiteaReleaseBoundary{:x}".format(secrets.randbits(64))
+    with open(filepath, "rb") as f:
+        payload = f.read()
+    body = (
+        "--{}\r\n"
+        'Content-Disposition: form-data; name="{}"; filename="{}"\r\n'
+        "Content-Type: application/octet-stream\r\n\r\n"
+    ).format(boundary, field, filename).encode() + payload + (
+        "\r\n--{}--\r\n".format(boundary)
+    ).encode()
+    return boundary, body
+
+
+def _api(method, url, headers, data=None, filepath=None):
+    req_headers = dict(headers)
+    body = None
+    if filepath:
+        boundary, body = _multipart_formdata("attachment", os.path.basename(filepath), filepath)
+        req_headers["Content-Type"] = "multipart/form-data; boundary={}".format(boundary)
+    elif data is not None:
+        body = json.dumps(data).encode()
+        req_headers.setdefault("Content-Type", "application/json")
+    req = urllib.request.Request(url, data=body, headers=req_headers, method=method)
+    try:
+        with urllib.request.urlopen(req) as resp:
+            return resp.status, json.loads(resp.read().decode())
+    except urllib.error.HTTPError as e:
+        snippet = e.read().decode()[:200] if e.fp else ""
+        print("  [release] HTTP {} on {} {}: {}".format(e.code, method, url, snippet))
+        return e.code, None
+
+
+def main():
+    # ── Guard ──
+    if env("ENABLE_RELEASE_UPLOAD").lower() not in ("true", "1", "yes"):
+        print("[release] ENABLE_RELEASE_UPLOAD not active — skipping.")
+        return
+
+    token = env("GITEA_TOKEN") or env("RELEASE_TOKEN")
+    if not token:
+        print("[release] No token (GITEA_TOKEN / RELEASE_TOKEN) — skipping.")
+        return
+
+    server_url = env("GITHUB_SERVER_URL").rstrip("/")
+    repo = env("GITHUB_REPOSITORY")
+    ref_name = env("GITHUB_REF_NAME")
+    sha = env("GITHUB_SHA")
+
+    if not repo:
+        print("[release] GITHUB_REPOSITORY unset — skipping.")
+        return
+
+    # ── Scan assets ──
+    assets_dir = sys.argv[1] if len(sys.argv) > 1 else "output"
+    whl_files = sorted(
+        glob.glob(os.path.join(assets_dir, "*.whl"))
+        + glob.glob(os.path.join(assets_dir, "wheels", "*.whl"))
+    )
+    zip_files = sorted(glob.glob(os.path.join(assets_dir, "*.zip")))
+    all_files = whl_files + zip_files
+
+    if not all_files:
+        print("[release] No .whl or .zip found under {}/ — nothing to upload.".format(assets_dir))
+        return
+
+    tag = ref_name or ("build-{}".format(sha[:7]) if sha else "latest")
+
+    # ── Gitea API ──
+    api_base = "{}/api/v1/repos/{}".format(server_url, repo)
+    headers = {"Authorization": "token {}".format(token), "Accept": "application/json"}
+
+    # 1) Get or create release
+    print("[release] Target tag: {}".format(tag))
+    status, release = _api("GET", "{}/releases/tags/{}".format(api_base, tag), headers)
+    if status == 200 and release:
+        release_id = release["id"]
+        print("[release] Found existing release #{} ({})".format(release_id, tag))
+    else:
+        print("[release] Creating release for tag {} ...".format(tag))
+        status, release = _api("POST", "{}/releases".format(api_base), headers, {
+            "tag_name": tag,
+            "target_commitish": sha,
+            "name": "EcoArk Wheels {}".format(tag),
+            "body": "Automated wheel build from commit {}.".format(sha[:7] if sha else "unknown"),
+            "draft": False,
+            "prerelease": True,
+        })
+        if status not in (200, 201):
+            print("[release] Failed to create release — skipping asset upload.")
+            return
+        release_id = release["id"]
+        print("[release] Created release #{} ({})".format(release_id, tag))
+
+    # 2) Upload each asset as an attachment
+    for fpath in all_files:
+        fname = os.path.basename(fpath)
+        print("[release] Uploading {} ...".format(fname))
+        _api("POST", "{}/releases/{}/assets".format(api_base, release_id), headers, filepath=fpath)
+
+    print('[release] Done — {} asset(s) uploaded to "{}".'.format(len(all_files), tag))
+
+
+if __name__ == "__main__":
+    main()