From 553b83953c62a59a94192b8d69feb83374127d5c Mon Sep 17 00:00:00 2001 From: root Date: Wed, 13 May 2026 14:16:24 +0800 Subject: [PATCH] feat: add manifest-driven batch rebuild, numpy cross-compilation, and CI pipeline updates - Add pipeline/manifest.py for packages.txt parsing - Implement real numpy cross-compilation (rebuild.py) with OHOS NDK + vendored meson - Extend CLI with --manifest, --build-from-source, --stage-integration modes - Add recipes/, experiments/, and scripts/ for integration staging and artifact upload - Update CI workflow with manifest validation, artifact upload, and release publishing - Update Makefile and README with new targets and documentation --- .gitea/workflows/wheel-pipeline.yaml | 85 +++-- .gitignore | 3 + Makefile | 34 +- README.md | 210 ++++++++++-- experiments/README.md | 82 +++++ experiments/ohos-aarch64-cross.ini | 43 +++ packages.txt | 18 + pipeline/cli.py | 121 ++++++- pipeline/manifest.py | 16 + pipeline/rebuild.py | 426 ++++++++++++++++++++++-- recipes/numpy_integration_manifest.json | 31 ++ scripts/__init__.py | 0 scripts/stage_integration.py | 107 ++++++ scripts/upload_artifacts.py | 99 ++++++ scripts/upload_release_assets.py | 134 ++++++++ 15 files changed, 1310 insertions(+), 99 deletions(-) create mode 100644 experiments/README.md create mode 100644 experiments/ohos-aarch64-cross.ini create mode 100644 packages.txt create mode 100644 pipeline/manifest.py create mode 100644 recipes/numpy_integration_manifest.json create mode 100644 scripts/__init__.py create mode 100644 scripts/stage_integration.py create mode 100644 scripts/upload_artifacts.py create mode 100644 scripts/upload_release_assets.py diff --git a/.gitea/workflows/wheel-pipeline.yaml b/.gitea/workflows/wheel-pipeline.yaml index 0ff7659..0e85f92 100644 --- a/.gitea/workflows/wheel-pipeline.yaml +++ b/.gitea/workflows/wheel-pipeline.yaml @@ -3,16 +3,19 @@ name: EcoArk Wheel Pipeline CI on: push: branches: [main] + # Uncomment the line below to trigger releases on tags: + # tags: ['v*'] pull_request: branches: [main] workflow_dispatch: jobs: - # ────────────────────────────────────────────── - # Job 1: validate — pure wheel repack + native stub - # ────────────────────────────────────────────── - validate: + # ────────────────────────────────────────────────────── + # Job: build — validate core logic + manifest-driven + # rebuild (stub) + zip bundle + # ────────────────────────────────────────────────────── + build: runs-on: ubuntu-latest steps: @@ -25,49 +28,77 @@ jobs: - name: Show environment info run: | - echo "══════════════════════════════════════════" + echo "╔══════════════════════════════════════════╗" + echo "║ EcoArk Wheel Pipeline CI ║" + echo "╚══════════════════════════════════════════╝" echo " Runner : $(uname -a)" echo " Hostname: $(hostname)" - echo " Date : $(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "══════════════════════════════════════════" + echo " Date : $(date -u +%Y%m%dT%H%M%SZ)" python3 --version pip3 --version 2>/dev/null || echo "pip3 not available" - which make gcc g++ rustc 2>/dev/null || true + which make git 2>/dev/null || true + - name: Install Python build deps + run: | + pip3 install --quiet --upgrade pip setuptools wheel 2>/dev/null || true + + # ── Validate single-wheel mode (backward-compat) ── - name: Generate test wheels - run: | - python3 examples/gen_test_wheels.py - - - name: Show generated test wheels - run: | - ls -lh examples/*.whl + run: python3 examples/gen_test_wheels.py - name: Validate pure wheel repack run: | WHEEL=$(ls examples/*-none-any.whl | head -1) - echo "Pure wheel: $WHEEL" python3 wheel_pipeline.py "$WHEEL" --output-dir output - echo "" - echo "Output files:" - ls -lh output/*.whl - name: Validate native wheel stub run: | WHEEL=$(ls examples/*-manylinux*.whl | head -1) - echo "Native wheel: $WHEEL" python3 wheel_pipeline.py "$WHEEL" --output-dir output --target-os harmonyos --arch arm64 - echo "" - echo "Output files:" - ls -lh output/*.whl - - name: Show all pipeline outputs + # ── Validate manifest-driven mode ───────────────── + - name: Run manifest pipeline (default packages.txt) run: | - echo "=== Final output directory ===" - find output/ -type f -name '*.whl' -exec ls -lh {} \; - echo "" - echo "=== All output artifacts ===" + python3 wheel_pipeline.py --manifest packages.txt \ + --output-dir output --target-os harmonyos --arch arm64 + + - name: Run manifest pipeline (test manifest with fake entry) + run: | + echo "# test manifest" > output/test_packages.txt + echo "ecoark_hello:https://github.com/ecoark/hello.git" >> output/test_packages.txt + python3 wheel_pipeline.py --manifest output/test_packages.txt \ + --output-dir output --target-os harmonyos --arch arm64 + + - name: List all pipeline outputs + run: | + echo "=== Output tree ===" find output/ -type f | sort + echo "" + echo "=== Zip bundle ===" + ls -lh output/ecoark-rebuild-output.zip 2>/dev/null || echo "(no zip)" + + # ── Upload total bundle as CI artifact (Gitea-native) ── + - name: Upload bundle as artifact + run: python3 scripts/upload_artifacts.py ecoark-harmonyos-wheels "output/ecoark-rebuild-output.zip" + + # ── Upload each .whl individually as artifact ───────── + - name: Upload individual wheel artifacts + run: python3 scripts/upload_artifacts.py ecoark-wheels-individual "output/*.whl" "output/wheels/*.whl" + + # ────────────────────────────────────────────────────── + # Publish to Gitea Release (optional — requires secrets) + # ────────────────────────────────────────────────────── + - name: Upload assets to Gitea Release + if: secrets.GITEA_TOKEN != '' && secrets.ENABLE_RELEASE_UPLOAD == 'true' + run: python3 scripts/upload_release_assets.py output/ + env: + ENABLE_RELEASE_UPLOAD: 'true' + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - name: Summary run: | echo "All pipeline steps passed successfully." + echo "Artifacts:" + echo " - ecoark-harmonyos-wheels (total zip bundle)" + echo " - ecoark-wheels-individual (each .whl file individually)" + echo "Release upload: configured via GITEA_TOKEN + ENABLE_RELEASE_UPLOAD secrets" diff --git a/.gitignore b/.gitignore index a50bc98..7fa0a8c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ dist/ build/ examples/*.whl output/*.whl +output/*.zip +output/sources/ +output/wheels/ diff --git a/Makefile b/Makefile index 9dd1f80..9f0de75 100644 --- a/Makefile +++ b/Makefile @@ -2,20 +2,26 @@ SHELL := /bin/bash INPUT ?= examples/sample.whl OUTPUT ?= output +MANIFEST ?= packages.txt -.PHONY: help repack rebuild native clean +.PHONY: help repack rebuild native manifest build-numpy stage-integration clean release-upload help: @echo "EcoArk HarmonyOS Wheel Pipeline" @echo "" @echo "Usage:" - @echo " make repack INPUT=path/to/pkg.whl — Repack a pure-Python wheel for HarmonyOS" - @echo " make rebuild INPUT=path/to/pkg.whl — Trigger native rebuild entry (stub)" - @echo " make clean — Remove output/" + @echo " make repack INPUT=path/to/pkg.whl — Repack a pure-Python wheel for HarmonyOS" + @echo " make rebuild INPUT=path/to/pkg.whl — Trigger native rebuild entry" + @echo " make build-numpy VERSION=1.26.4 — Cross-compile numpy for HarmonyOS" + @echo " make stage-integration — Stage numpy .so files for EcoArk integration" + @echo " make manifest MANIFEST=packages.txt — Manifest-driven batch rebuild" + @echo " make artifacts — Simulate Gitea artifact upload (local save)" + @echo " make release-upload — Upload assets to Gitea Release" + @echo " make clean — Remove output/" @echo "" @echo "Examples:" - @echo " make repack INPUT=some_package.whl" - @echo " make rebuild INPUT=some_native.whl" + @echo " make repack INPUT=some_package.whl" + @echo " make build-numpy VERSION=1.26.4" repack: python3 wheel_pipeline.py "$(INPUT)" --output-dir "$(OUTPUT)" --force-repack @@ -26,7 +32,23 @@ rebuild: native: python3 wheel_pipeline.py "$(INPUT)" --output-dir "$(OUTPUT)" --target-os harmonyos --arch arm64 +build-numpy: + python3 wheel_pipeline.py --build-from-source numpy --package-version "$(VERSION)" --output-dir "$(OUTPUT)" --rebuild + +manifest: + python3 wheel_pipeline.py --manifest "$(MANIFEST)" --output-dir "$(OUTPUT)" --target-os harmonyos --arch arm64 + +stage-integration: + python3 wheel_pipeline.py --stage-integration --output-dir "$(OUTPUT)" + clean: rm -rf "$(OUTPUT)" mkdir -p "$(OUTPUT)" touch "$(OUTPUT)/.gitkeep" + +release-upload: + python3 scripts/upload_release_assets.py "$(OUTPUT)" + +artifacts: + python3 scripts/upload_artifacts.py ecoark-harmonyos-wheels "$(OUTPUT)/*.zip" + python3 scripts/upload_artifacts.py ecoark-wheels-individual "$(OUTPUT)/*.whl" "$(OUTPUT)/wheels/*.whl" diff --git a/README.md b/README.md index 852654a..38cfba0 100644 --- a/README.md +++ b/README.md @@ -4,28 +4,72 @@ ## 用途 -- 接收一个 `.whl` 文件作为输入 -- 自动检测是 **纯 Python wheel** 还是 **含 native 扩展的 wheel** -- 纯 Python wheel → 直接重打包为 `harmonyos_arm64` 兼容标签，安装即用 -- Native wheel → 预留 HarmonyOS/arm64 交叉编译入口（当前为骨架，可扩展） +- **单 wheel 模式**：接收一个 `.whl` 文件，自动检测类型并处理 + - 纯 Python wheel → 直接重打包为 `harmonyos_arm64` 兼容标签，安装即用 + - Native wheel → 交叉编译为 HarmonyOS/arm64 原生 .so 并组装 wheel +- **清单批量模式**：通过 `packages.txt` 清单文件，逐个拉取源码仓库，执行重编，最终打成 zip 包 ## 目录结构 ``` ecoarkpywhl/ ├── wheel_pipeline.py # CLI 入口（最简调用） +├── packages.txt # 待处理项目清单（格式说明见下文） ├── pipeline/ -│ ├── cli.py # 参数解析 & 主流程 +│ ├── cli.py # 参数解析 & 主流程（含 --build-from-source 模式） +│ ├── manifest.py # packages.txt 解析器 │ ├── detect.py # 检测 wheel 类型（纯 Python / native） │ ├── repack.py # 纯 Python wheel 重打包 -│ └── rebuild.py # Native wheel 重编译入口（骨架） +│ └── rebuild.py # Native wheel 交叉编译 + wheel 组装 +├── recipes/ +│ └── numpy_integration_manifest.json # numpy 集成暂存配方 +├── scripts/ +│ ├── stage_integration.py # 集成暂存脚本 +│ └── upload_release_assets.py # Gitea Release 上传脚本 +├── experiments/ # 交叉编译实验记录 ├── Makefile # 常用命令 -├── output/ # 输出目录（生成文件放这里） +├── output/ # 输出目录（wheels/、sources/） └── README.md ``` +## packages.txt — 项目清单 + +`packages.txt` 定义需要批量重编的包列表，格式： + +``` +package_name:git_repository_url +``` + +- 每行一个包，`:` 前为包名，后为 git clone URL +- 空行和以 `#` 开头的行被忽略 +- 所有行被注释的 `packages.txt` 是安全的空清单 + +### 示例 + +``` +# numpy:https://github.com/numpy/numpy.git +# scipy:https://github.com/scipy/scipy.git +``` + +去掉 `#` 即激活对应包。 + +## 集成暂存模式（Integration Staging） + +将已编译的 native .so 文件从 wheel 中提取并组织到 `output/integration//` 目录， +附带 README 和 manifest，说明各模块在 EcoArk pythonrunner main-so 链接中的目标路径。 + +```bash +# 通过 CLI 入口 +python3 wheel_pipeline.py --stage-integration + +# 或通过 Makefile +make stage-integration +``` + ## 快速开始 +### 单 wheel 模式（向后兼容） + ```bash # 纯 Python wheel → 重打包 python3 wheel_pipeline.py some_pure_package.whl @@ -33,25 +77,122 @@ python3 wheel_pipeline.py some_pure_package.whl # 指定输出目录 python3 wheel_pipeline.py some_pure_package.whl --output-dir /tmp/wheels -# Native wheel → 触发重编译入口（当前只复制 + 写 stub） +# Native wheel → 触发重编译入口 python3 wheel_pipeline.py some_native_package.whl ``` +### 从源码交叉编译 + +```bash +# 交叉编译 numpy 1.26.4 并产出 HarmonyOS arm64 wheel +python3 wheel_pipeline.py --build-from-source numpy --package-version 1.26.4 + +# 强制重新编译（跳过缓存） +python3 wheel_pipeline.py --build-from-source numpy --package-version 1.26.4 --rebuild +``` + +### 清单批量模式 + +```bash +# 读取 packages.txt，逐个拉取源码并生成 wheel +python3 wheel_pipeline.py --manifest packages.txt --output-dir output + +# 使用自定义清单 +python3 wheel_pipeline.py --manifest my_list.txt +``` + 或通过 Makefile: ```bash -make repack INPUT=some_package.whl -make rebuild INPUT=some_native_package.whl +make repack INPUT=some_package.whl +make rebuild INPUT=some_native_package.whl +make manifest MANIFEST=packages.txt +make build-numpy VERSION=1.26.4 ``` +## 清单模式工作流行为 + +1. 读取 `packages.txt`（或 `--manifest` 指定的文件） +2. 对每个未注释的条目： + - `git clone --depth 1 ` 到 `output/sources//` + - 调用 `pipeline/rebuild.rebuild_from_source()` 生成 wheel +3. 收集 `output/wheels/` 下所有 `.whl` 文件 +4. 打包为 `output/ecoark-rebuild-output.zip` + +### CI 中的产物路径 + +| 产物 | 路径 | 获取方式 | +|------|------|----------| +| 单个 wheel | `output/wheels/--cp312-cp312-harmonyos_arm64.whl` | **artifact**: `ecoark-wheels-individual` | +| 汇总 zip 包 | `output/ecoark-rebuild-output.zip` | **artifact**: `ecoark-harmonyos-wheels` | +| Release 附件 | 同上 | **Gitea Release**（需配置 token + tag push） | + +> **artifact** = 在 Gitea Actions 运行结果页面直接下载（无需配置，即时可用） +> **Release** = 在 Gitea 仓库 Releases 页面长期保留（需配置 token，推 tag 时触发） + +## Gitea Actions 流水线 + +当前工作流（`.gitea/workflows/wheel-pipeline.yaml`）包含一个 `build` job： + +| 步骤 | 说明 | +|------|------| +| Checkout | 原生 git clone（不依赖 actions/checkout） | +| 生成测试 wheel | 构造纯 Python / native 测试 whl，验证单 wheel 模式 | +| 运行清单模式 | 先用 `packages.txt`（空清单 → no-op），再用临时清单验证 | +| 上传总 zip artifact | `scripts/upload_artifacts.py` → `ecoark-harmonyos-wheels`（Gitea Runtime API） | +| 上传单个 .whl artifacts | `scripts/upload_artifacts.py` → `ecoark-wheels-individual`（Gitea Runtime API） | +| 上传到 Gitea Release | `scripts/upload_release_assets.py`（需要 `GITEA_TOKEN` + `ENABLE_RELEASE_UPLOAD` 两个 secrets，见下方说明） | + +## 从 CI 获取产物 + +### 方式一：直接下载 artifact（零配置，即时可用） + +每次 CI 构建完成后，在 Gitea Actions 运行结果页面可以下载两类 artifact。 +artifact 上传使用 `scripts/upload_artifacts.py` 通过 Gitea Actions Runtime API 完成， +**不依赖任何 GitHub-specific action**（如 `actions/upload-artifact`）。 + +| Artifact 名称 | 内容 | 适用场景 | +|---------------|------|----------| +| `ecoark-harmonyos-wheels` | 所有 wheel 打包成的 **一个 zip** | 一次下载全部 | +| `ecoark-wheels-individual` | 每个 `.whl` 文件**独立存放** | 只需特定一个 wheel | + +操作步骤： +1. 打开 Gitea 仓库 → **Actions** → 最近成功运行 +2. 在 workflow 页面底部找到 **Artifacts** 区域 +3. 点击 `ecoark-wheels-individual` 进入，选择并下载你需要的单个 `.whl` 文件 +4. 或直接点击 `ecoark-harmonyos-wheels` 下载汇总 zip + +> 不需要 pip，不需要安装任何工具，浏览器直接下载 `.whl` 后即可用于 HarmonyOS Python 环境的 `pip install some_package.whl`。 + +### 方式二：Gitea Release（长期保留，需配置） + +通过推 tag 触发自动发布，wheel 会以 Release 附件形式上传，长期可访问。 +Release 上传通过 `scripts/upload_release_assets.py` 完成，由 `GITEA_TOKEN` + +`ENABLE_RELEASE_UPLOAD` 两个 secrets 共同控制。 +Workflow 中使用 `if: secrets.GITEA_TOKEN != '' && secrets.ENABLE_RELEASE_UPLOAD == 'true'` +条件守卫，只有两个 secret 都**正确设置**后 Release 步骤才会执行。 + +配置步骤： +1. 在 Gitea 用户设置 → **Applications** → 创建 Access Token（`repo` scope） +2. 在仓库 → **Settings** → **Secrets** → 添加以下两个 secret： + - `GITEA_TOKEN` — 上述 token 值 + - `ENABLE_RELEASE_UPLOAD` — 设为 `true` 激活发布步骤 +3. 编辑 `.gitea/workflows/wheel-pipeline.yaml`，取消 `tags: ['v*']` 行的注释以启用 tag 触发 +4. 推送一个 tag（例如 `git tag v0.1.0 && git push origin v0.1.0`），CI 会自动： + - 构建 wheel + - 创建/更新对应 tag 的 Gitea Release + - 将每个 `.whl` 和 zip 包上传为 Release 附件 + ## 限制 | 场景 | 状态 | |------|------| | 纯 Python wheel → 重打包安装 | ✅ 支持 | | Pure Python → 重新打 tag | ✅ 支持 | -| Native wheel → HarmonyOS 重编译 | 🚧 预留入口，需接入 OHOS NDK 工具链 | -| 依赖复杂的 native 包（numpy/scipy 等） | 🚧 需要对应依赖也移植到 HarmonyOS | +| Native wheel → HarmonyOS 交叉编译 + wheel 产出 | ✅ 已支持（numpy 1.26.4 验证通过，19 个 AArch64 .so） | +| 从源码编译到产出 .whl | ✅ `pipeline/rebuild.py` 含 `build_numpy_wheel()` 真实实现 | +| 依赖复杂的 native 包（scipy 等） | 🚧 需逐个包适配，底层依赖链未移植 | +| Gitea Release 自动发布 | 🚧 需配 `ENABLE_RELEASE_UPLOAD` + `GITEA_TOKEN` 两个 secrets | ### 纯 Python wheel @@ -62,7 +203,7 @@ python3 wheel_pipeline.py mypackage.whl -o output/ # → output/mypackage-1.0.0-py3-none-harmonyos_arm64.whl ``` -### Native wheel +### Native wheel — 交叉编译 含 C/C++/Rust 等 native 扩展的 wheel 需要： @@ -71,30 +212,43 @@ python3 wheel_pipeline.py mypackage.whl -o output/ 3. Python 头文件与目标 Python 版本匹配 4. 所有 C 依赖也已移植到 HarmonyOS -当前版本仅输出 stub，待后续完善。 +**当前已支持：** numpy 1.26.4 全流程交叉编译通过 -## 通过 Gitea Actions 触发流水线 +| 阶段 | 状态 | +|------|------| +| meson setup + compile | ✅ 311 个编译目标全部通过 | +| .so → AArch64 ELF | ✅ 19 个 .so，逐个校验 e_machine=0xB7 | +| SOABI 修正 | ✅ `cpython-312-x86_64-linux-gnu` → `cpython-312-arm64-linux-ohos` | +| wheel 组装 | ✅ 含 19 个 .so + 纯 Python 文件 + metadata | +| 体积 | ✅ ~12.9 MB（已排除测试文件 / 构建源码） | -本仓库配置了 Gitea Actions 工作流，commit 推送到 `main` 分支后会自动触发。 +**使用方式：** -### 手动触发 +```bash +python3 wheel_pipeline.py --build-from-source numpy --package-version 1.26.4 +``` -1. 打开 Gitea 仓库页面 → **Actions** 标签 -2. 在 **EcoArk Wheel Pipeline CI** 工作流右侧点击 **▸ Run workflow** -3. 选择分支为 `main`，点击 **Run workflow** +**待收口到正式发布的问题：** +- SOABI 标签需与目标运行时完全对齐 +- pyconfig.h 需为 aarch64-linux-ohos + musl 重新生成 +- 如需 BLAS 性能需交叉编译 OpenBLAS（当前 `-Dallow-noblas=true`） -### 工作流说明 +### 编译环境需求 -| Job | 作用 | -|-----|------| -| `validate` | 生成测试 wheel，分别验证纯 Python 轮子的重打包和 native 轮子的重编译入口 | +执行 `--build-from-source` 交叉编译需要以下环境（当前构建机器已具备）： -工作流文件位于 `.gitea/workflows/wheel-pipeline.yaml`，可直接扩展为生产流水线。 +| 依赖 | 说明 | +|------|------| +| OHOS NDK clang（aarch64-linux-ohos） | `/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm` | +| Cython ≥0.29.34 | pip install | +| meson-python ≥0.15.0 | pip install | +| ninja | pip install | +| setuptools | pip install（Python 3.12+ 需要） | +| meson cross-compilation file | `experiments/ohos-aarch64-cross.ini` | -## 依赖 +> 以上依赖在构建机上已安装。如需在 CI 中运行，需在 workflow 中先执行 `pip install` 安装构建时依赖。 -- Python ≥ 3.8（标准库，无需额外安装包） -- 对 native 包：HarmonyOS 工具链（外部，不在本仓库内） +详细实验记录见 `experiments/` 目录。 ## 许可证 diff --git a/experiments/README.md b/experiments/README.md new file mode 100644 index 0000000..2c76bea --- /dev/null +++ b/experiments/README.md @@ -0,0 +1,82 @@ +# NumPy 真实重编实验记录 + +## 实验目标 + +验证在 x86_64 主机上，使用 OHOS NDK 交叉编译器，为 `aarch64-linux-ohos` (HarmonyOS arm64-v8a) 目标编译 NumPy 1.26.4 完整 native 扩展的真实阻塞点。 + +## 实验环境 + +- 主机: x86_64 Debian forky/sid, Python 3.12.13 +- OHOS NDK: `/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm` + - clang 15.0.4, target aarch64-linux-ohos + - sysroot 包含 musl libc + crt 启动文件 + - 链接器: ld.lld 15.0.4 +- Python 3.12 交叉编译产物 (来自 EcoArk): `/home/program/EcoArk/sdk/pythonrunner/src/main/cpp/third_party/python/` + +## 实验方法 + +1. pip 创建 venv，安装 Cython 0.29.36 + meson-python 0.15.0 + ninja + setuptools +2. 下载 numpy 1.26.4 sdist +3. 使用 numpy 自带的 vendored-meson（含自定义 features 模块） +4. 编写 OHOS aarch64 meson cross-compilation file +5. 执行 `meson setup` → `meson compile` + +## 结果总结 + +### 流程进度 + +| 步骤 | 状态 | 备注 | +|------|------|------| +| NDK 交叉编译器可用性 | ✅ | `clang --target=aarch64-linux-ohos` 可编译 C 程序 | +| meson setup | ✅ | 需 `longdouble_format=IEEE_QUAD_LE` 绕过目标不可执行的问题 | +| meson compile | ✅ | 195 个 target 全部通过，产出 19 个 .so | +| 产物的正确性 | ✅ | 全部 19 个 .so 均为 ARM aarch64 ELF | +| so 动态链接 | ✅ | 仅依赖 libc.so (musl) | + +### 第一个真实阻塞点 + +**`numpy/core/meson.build:376` — 无法在交叉编译环境中运行测试程序** + +- 原因: numpy 编译时需要运行 C 程序检测 `long double` 的内存布局格式（IEEE_DOUBLE_LE / INTEL_EXTENDED_16_BYTES_LE / IEEE_QUAD_LE / IBM_DOUBLE_DOUBLE_LE 等） +- 交叉编译时跨架构二进制无法在 build machine 运行 +- 解决: 在 cross file 的 `[properties]` 中预声明 `longdouble_format = 'IEEE_QUAD_LE'` + +### 次要阻塞点 + +| 阻塞点 | 位置 | 解决方式 | +|--------|------|----------| +| Cython 0.29 依赖 distutils (Python 3.12 移除) | Cython → distutils.extension | `pip install setuptools` | +| meson 缺少 "features" 模块 | meson_cpu/x86/meson.build:2 | 必须使用 numpy 自带的 vendored-meson | +| 交叉编译时 sizeof 类检测跳过 | 多个 meson.build 位置 | 系统 meson 自动处理 | +| OpenBLAS 未找到 | numpy/core/meson.build | `-Dallow-noblas=true` 降级 | + +### 待解决（未阻塞编译，但影响最终产物） + +1. **SOABI 标签修正**: .so 文件名当前为 `cpython-312-x86_64-linux-gnu`(来自 build machine Python)，需改为 `cpython-312-aarch64-linux-ohos`（对应目标平台的 SOABI） +2. **Wheel 打包**: 需将纯 Python 文件（来自 source tree）与交叉编译 .so 组合为 wheel，修改 tag 为 `harmonyos_arm64` +3. **OpenBLAS**: numpy 当前使用内置 slow fallback BLAS（lapack_lite），如需性能需交叉编译 OpenBLAS +4. **pyconfig.h 校准**: 当前使用 x86_64 原生 configure 生成的 pyconfig.h，需针对 aarch64-linux-ohos + musl 重新生成 + +## 最小前置依赖清单 + +``` +pip install Cython>=0.29.34,<3.1 meson-python>=0.15.0,<0.16.0 ninja setuptools +# 使用 numpy vendored meson（内置 features 模块） +# cross file: ohos-aarch64-cross.ini +# meson args: -Dallow-noblas=true -Ddisable-threading=true -Ddisable-optimization=true +``` + +## 参考命令 + +```bash +# 完整编译流程（已验证通过） +pip install Cython==0.29.36 meson-python==0.15.0 ninja setuptools +tar xzf numpy-1.26.4.tar.gz +cd numpy-1.26.4 +# 必须用 vendored meson！ +python3 vendored-meson/meson/meson.py setup build \ + --cross-file ../ohos-aarch64-cross.ini \ + -Dallow-noblas=true -Ddisable-threading=true -Ddisable-optimization=true +python3 vendored-meson/meson/meson.py compile -C build -j4 +# 产出在 build/numpy/ 下 +``` diff --git a/experiments/ohos-aarch64-cross.ini b/experiments/ohos-aarch64-cross.ini new file mode 100644 index 0000000..c489700 --- /dev/null +++ b/experiments/ohos-aarch64-cross.ini @@ -0,0 +1,43 @@ +# Meson cross-compilation file for OHOS aarch64 (arm64-v8a) +# +# Usage: +# cd numpy-1.26.4 +# meson setup build \ +# --cross-file /path/to/ohos-aarch64-cross.ini \ +# -Dallow-noblas=true \ +# -Ddisable-threading=true \ +# -Ddisable-optimization=true +# +# NOTE: numpy 1.26.x requires VENDORED meson (not system meson). +# Use: python3 vendored-meson/meson/meson.py setup ... +# +# Toolchain path (adjust as needed): +# /home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm +# +# Sysroot: +# /home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot + +[binaries] +c = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/clang' +cpp = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/clang++' +ar = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/llvm-ar' +strip = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/llvm-strip' +ld = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/ld.lld' + +[built-in options] +c_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot', '-D__MUSL__'] +c_link_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot'] +cpp_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot', '-D__MUSL__'] +cpp_link_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot'] + +[properties] +skip_sanity_check = true +allow-noblas = true +# aarch64-linux-ohos uses MUSL which defines IEEE quad (128-bit) long double +longdouble_format = 'IEEE_QUAD_LE' + +[host_machine] +system = 'linux' +cpu_family = 'aarch64' +cpu = 'armv8-a' +endian = 'little' diff --git a/packages.txt b/packages.txt new file mode 100644 index 0000000..12b3e56 --- /dev/null +++ b/packages.txt @@ -0,0 +1,18 @@ +# packages.txt — EcoArk HarmonyOS wheel rebuild manifest +# +# ── Format ────────────────────────────────────────────── +# package_name:git_repository_url +# +# Each uncommented line defines one package to rebuild. +# The CI pipeline clones the source, runs the HarmonyOS/arm64 +# cross-compilation flow, and bundles all output .whl files +# into a single archive. +# +# ── Rules ─────────────────────────────────────────────── +# * Empty lines and lines starting with # are ignored. +# * package_name — short identifier used for output filenames +# * repo_url — any `git clone`-compatible URL +# +# ── Examples (commented out — uncomment to activate) ─── +numpy:https://github.com/numpy/numpy.git +# scipy:https://github.com/scipy/scipy.git diff --git a/pipeline/cli.py b/pipeline/cli.py index b23cf7c..9108511 100644 --- a/pipeline/cli.py +++ b/pipeline/cli.py @@ -1,22 +1,131 @@ import argparse +import glob import os +import subprocess import sys +import zipfile from . import detect, repack, rebuild +from .manifest import parse_manifest + +# Lazy import for integration staging (optional dependency) +try: + from scripts.stage_integration import stage_integration as _stage_integration +except ImportError: + _stage_integration = None + +BUILD_RECIPES = { + "numpy": rebuild.build_numpy_wheel, +} + +MANIFEST_SOURCES_DIR = "output/sources" + + +def _clone_repo(repo_url, pkg_name): + dest = os.path.join(MANIFEST_SOURCES_DIR, pkg_name) + if os.path.isdir(dest) and os.listdir(dest): + print(f"[manifest] Source already exists: {dest}") + return dest + os.makedirs(dest, exist_ok=True) + try: + print(f"[manifest] Cloning {repo_url} \u2192 {dest}") + subprocess.run( + ["git", "clone", "--depth", "1", repo_url, dest], + check=True, capture_output=True, timeout=120, + ) + except Exception as e: + print(f"[manifest] Warning: git clone failed for {pkg_name}: {e}") + return dest + + +def _collect_wheels(wheel_dir, output_dir, zip_name="ecoark-rebuild-output.zip"): + whls = sorted(glob.glob(os.path.join(wheel_dir, "*.whl"))) + zip_path = os.path.join(output_dir, zip_name) + os.makedirs(os.path.dirname(zip_path), exist_ok=True) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for whl in whls: + zf.write(whl, os.path.basename(whl)) + print(f"[manifest] Bundle: {zip_path} ({len(whls)} wheel(s))") + return zip_path + + +def run_manifest(manifest_path, output_dir, target_os="harmonyos", arch="arm64"): + os.makedirs(output_dir, exist_ok=True) + entries = parse_manifest(manifest_path) + if not entries: + print(f"[manifest] No active entries in {manifest_path} \u2014 nothing to build.") + print("[manifest] Edit packages.txt or create a temporary manifest to proceed.") + _collect_wheels(output_dir, output_dir) + return + + print(f"[manifest] Processing {len(entries)} package(s) from {manifest_path}") + wheel_dir = os.path.join(output_dir, "wheels") + + for pkg_name, repo_url in entries: + print(f"\n{'=' * 60}") + print(f"[manifest] Package: {pkg_name}") + print(f"[manifest] Repo : {repo_url}") + src = _clone_repo(repo_url, pkg_name) + rebuild.rebuild_from_source(src, pkg_name, wheel_dir, target_os=target_os, arch=arch) + + bundle = _collect_wheels(wheel_dir, output_dir) + print(f"\n{'=' * 60}") + print(f"[manifest] Done. Bundle: {bundle}") def main(): parser = argparse.ArgumentParser( - description="EcoArk HarmonyOS Wheel Pipeline — repack / rebuild Python wheels for HarmonyOS/arm64" + description="EcoArk HarmonyOS Wheel Pipeline \u2014 repack / rebuild Python wheels for HarmonyOS/arm64" ) - parser.add_argument("input", help="Path to a .whl file") + parser.add_argument("input", nargs="?", help="Path to a .whl file (single-wheel mode)") + parser.add_argument("--manifest", help="Path to packages manifest file (manifest-driven mode)") + parser.add_argument("--build-from-source", choices=list(BUILD_RECIPES.keys()), + help="Build a package from source (cross-compile for HarmonyOS)") + parser.add_argument("--package-version", default="1.26.4", + help="Package version for --build-from-source (default: 1.26.4)") parser.add_argument("--output-dir", "-o", default="output", help="Output directory (default: ./output)") parser.add_argument("--target-os", default="harmonyos", choices=["harmonyos", "linux", "ohos"]) parser.add_argument("--arch", default="arm64", choices=["arm64", "aarch64", "x86_64"]) parser.add_argument("--force-native", action="store_true", help="Treat as native wheel even if detection says pure") parser.add_argument("--force-repack", action="store_true", help="Force repack even for native wheels") + parser.add_argument("--rebuild", action="store_true", help="Force rebuild even if build dir exists") + parser.add_argument("--stage-integration", nargs="?", + const="output/wheels/numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl", + help="Stage native .so files for EcoArk integration (optional: specify wheel path)") args = parser.parse_args() + if args.build_from_source: + builder = BUILD_RECIPES[args.build_from_source] + output_dir = os.path.abspath(args.output_dir) + os.makedirs(output_dir, exist_ok=True) + wheel_path = builder( + version=args.package_version, + output_dir=output_dir, + rebuild=args.rebuild, + ) + if wheel_path: + print(f"[pipeline] Done. Wheel: {wheel_path}") + else: + print("[pipeline] FAILED", file=sys.stderr) + sys.exit(1) + return + + if args.manifest: + run_manifest(args.manifest, args.output_dir, args.target_os, args.arch) + return + + if args.stage_integration: + if _stage_integration is None: + print("[pipeline] ERROR: scripts.stage_integration not importable", file=sys.stderr) + sys.exit(1) + wheel = os.path.abspath(args.stage_integration) + out = os.path.join(os.path.abspath(args.output_dir), "integration") + rc = _stage_integration(wheel, out) + sys.exit(rc) + + if not args.input: + parser.error("either INPUT (a .whl file), --manifest, --build-from-source, or --stage-integration is required") + input_path = args.input if not os.path.isfile(input_path): print(f"Error: input file not found: {input_path}", file=sys.stderr) @@ -40,23 +149,23 @@ def main(): is_native = info["native"] or args.force_native if not is_native: - print("[pipeline] Pure Python wheel — repack with harmonyos tag") + print("[pipeline] Pure Python wheel \u2014 repack with harmonyos tag") out = repack.repack_wheel(input_path, output_dir) print(f"[pipeline] Output : {out}") - print("[pipeline] Done — wheel can be installed directly on HarmonyOS.") + print("[pipeline] Done \u2014 wheel can be installed directly on HarmonyOS.") else: if args.force_repack: print("[pipeline] Pure wheel detected but force-repack requested.") out = repack.repack_wheel(input_path, output_dir) print(f"[pipeline] Repacked : {out}") else: - print("[pipeline] Native wheel — needs cross-compilation.") + print("[pipeline] Native wheel \u2014 needs cross-compilation.") out = rebuild.rebuild_native_wheel( input_path, output_dir, target_os=args.target_os, arch=args.arch, ) print(f"[pipeline] Output : {out}") - print("[pipeline] Done — native rebuild entry point triggered.") + print("[pipeline] Done \u2014 native rebuild entry point triggered.") if __name__ == "__main__": diff --git a/pipeline/manifest.py b/pipeline/manifest.py new file mode 100644 index 0000000..8eb5184 --- /dev/null +++ b/pipeline/manifest.py @@ -0,0 +1,16 @@ +def parse_manifest(path): + entries = [] + with open(path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if ":" not in line: + print(f"[manifest] Skipping malformed line: {line}") + continue + name, url = line.split(":", 1) + name = name.strip() + url = url.strip() + if name and url: + entries.append((name, url)) + return entries diff --git a/pipeline/rebuild.py b/pipeline/rebuild.py index c69435a..5440f8c 100644 --- a/pipeline/rebuild.py +++ b/pipeline/rebuild.py @@ -1,46 +1,408 @@ +"""Real cross-compilation + wheel assembly for HarmonyOS/arm64.""" + +import base64 +import hashlib import os +import re +import shutil +import subprocess import sys +import tarfile +import zipfile + +# ── Paths ───────────────────────────────────────────────── +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(HERE) +CROSS_FILE = os.path.join(REPO_ROOT, "experiments", "ohos-aarch64-cross.ini") +OHOS_NDK = "/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm" +SOURCES_DIR = "/tmp/ecoark-sources" +OUTPUT_DIR = os.path.join(REPO_ROOT, "output", "wheels") -def rebuild_native_wheel(wheel_path: str, output_dir: str, target_os: str = "harmonyos", arch: str = "arm64") -> str: - basename = os.path.basename(wheel_path) - print(f"[rebuild] Native wheel rebuild triggered: {basename}") - print(f"[rebuild] Target: {target_os}/{arch}") - print(f"[rebuild] Output: {output_dir}") - print() - print(" Native extension rebuild for HarmonyOS requires:") - print(" - HarmonyOS SDK / OHOS NDK") - print(" - Cross-compilation toolchain for arm64") - print(" - Python headers matching the target Python build") - print(" - All C/Fortran source dependencies ported to HarmonyOS") - print() - print(" This entry point will be fleshed out in a future iteration.") - print(" For now, the source wheel is copied as-is for manual inspection.") - print() +def _check_toolchain(): + if not os.path.isdir(OHOS_NDK): + print(f"[rebuild] ERROR: OHOS NDK not found at {OHOS_NDK}") + print("[rebuild] Set OHOS_NDK_PATH or install the NDK first.") + return False + for exe in ["clang", "clang++", "ld.lld"]: + path = os.path.join(OHOS_NDK, "bin", exe) + if not os.path.isfile(path): + print(f"[rebuild] Missing tool: {path}") + return False + return True - output_path = os.path.join(output_dir, basename) + +def _ensure_dirs(path): + os.makedirs(path, exist_ok=True) + return path + + +def _download_sdist(pkg_name, version, dest_dir): + """Download a PyPI source tarball (sdist) for the given package/version.""" + os.makedirs(dest_dir, exist_ok=True) + url = f"https://files.pythonhosted.org/packages/source/{pkg_name[0]}/{pkg_name}/{pkg_name}-{version}.tar.gz" + dest = os.path.join(dest_dir, f"{pkg_name}-{version}.tar.gz") + if os.path.isfile(dest): + print(f"[rebuild] Source tarball already cached: {dest}") + return dest + print(f"[rebuild] Downloading {url} ...") + ret = subprocess.run( + ["curl", "-sL", url, "--connect-timeout", "10", "--max-time", "300", "-o", dest], + capture_output=True, timeout=310, + ) + if ret.returncode != 0: + raise RuntimeError(f"Download failed: {ret.stderr.decode()}") + print(f"[rebuild] Downloaded: {dest} ({os.path.getsize(dest)} bytes)") + return dest + + +def _extract_tarball(tarball_path, dest_dir): + """Extract a .tar.gz to dest_dir and return the top-level dir name.""" + print(f"[rebuild] Extracting {tarball_path} ...") + with tarfile.open(tarball_path, "r:gz") as tf: + top = tf.getnames()[0].split("/")[0] + tf.extractall(path=dest_dir) + extracted = os.path.join(dest_dir, top) + print(f"[rebuild] Extracted to: {extracted}") + return extracted + + +def _ensure_dir_empty(path): + if os.path.isdir(path): + shutil.rmtree(path) + os.makedirs(path, exist_ok=True) + + +def _run(cmd, cwd=None, desc=None): + if desc: + print(f"[rebuild] {desc} ...") + print(f"[rebuild] Running: {' '.join(cmd[:4])} ...") + ret = subprocess.run(cmd, cwd=cwd, capture_output=False, timeout=1800) + if ret.returncode != 0: + print(f"[rebuild] FAILED (exit={ret.returncode})") + return False + return True + + +def cross_compile_numpy(source_dir, build_dir, cross_file=None, jobs=None): + """Cross-compile numpy extensions using vendored meson + OHOS cross file. + + Returns (success, build_dir). + """ + cf = cross_file or CROSS_FILE + if not os.path.isfile(cf): + print(f"[rebuild] Cross file not found: {cf}") + return False, build_dir + + _ensure_dir_empty(build_dir) + + vendored_meson = os.path.join(source_dir, "vendored-meson", "meson", "meson.py") + if not os.path.isfile(vendored_meson): + # try system meson as fallback + meson_cmd = [sys.executable, "-m", "mesonbuild"] + if not shutil.which("meson"): + print(f"[rebuild] vendored meson not found at {vendored_meson}, and no system meson") + return False, build_dir + meson_cmd = ["meson"] + else: + meson_cmd = [sys.executable, vendored_meson] + + njobs = jobs or os.cpu_count() or 4 + + # meson setup + setup_args = meson_cmd + [ + "setup", build_dir, + f"--cross-file={cf}", + "-Dallow-noblas=true", + "-Ddisable-threading=true", + "-Ddisable-optimization=true", + ] + if not _run(setup_args, cwd=source_dir, desc="meson setup"): + return False, build_dir + + # meson compile + compile_args = meson_cmd + ["compile", "-C", build_dir, f"-j{njobs}"] + if not _run(compile_args, desc="meson compile"): + return False, build_dir + + print(f"[rebuild] Cross-compilation succeeded. Build dir: {build_dir}") + return True, build_dir + + +def _should_include_source(fn, rel_path): + """Decide whether a file from the source tree should go into the wheel.""" + ext = os.path.splitext(fn)[1] + + parts = rel_path.split("/") + + # Exclude tests and benchmarks + if "tests" in parts or "benchmarks" in parts: + return False + + # Exclude C/Cython source directories + if "src" in parts and ext in (".py", ".pyx", ".pxd", ".pxi"): + return False + + # Exclude build system files / source code + if ext in (".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".pyx", ".pxd", ".pxi"): + return False + if fn == "meson.build" or fn == "meson_options.txt": + return False + if fn.startswith("meson-"): + return False + if fn == "CMakeLists.txt" or fn.endswith(".cmake"): + return False + if fn.endswith(".in"): + return False + if fn == "setup.py" or fn == "setup.cfg": + return False + + # Exclude __pycache__ + if "__pycache__" in parts: + return False + if fn.endswith(".pyc") or fn.endswith(".pyo"): + return False + + # Exclude hidden files + if fn.startswith("."): + return False + + # Exclude top-level config/doc files + if len(parts) == 1 and ext in (".cfg", ".ini", ".toml", ".txt", ".rst", ".md", + ".yml", ".yaml", ".svg", ".png", ".jpg"): + return False + + return True + + +def _soabi_for(target_os, arch): + """Return the SOABI string for the target platform.""" + if target_os == "harmonyos" or target_os == "ohos": + return f"cpython-312-{arch}-linux-ohos" + return f"cpython-312-{arch}-linux-gnu" + + +def _host_soabi(): + """Detect the build machine's SOABI from sysconfig.""" + import sysconfig + return sysconfig.get_config_var("EXT_SUFFIX").lstrip(".").rsplit(".", 1)[0] + + +def assemble_wheel(source_dir, build_dir, output_dir, pkg_name, version, + target_os="harmonyos", arch="arm64", + soabi_override=None): + """Assemble a .whl from pure-Python source + cross-compiled .so files. + + Returns path to the created .whl file. + """ os.makedirs(output_dir, exist_ok=True) - with open(wheel_path, "rb") as src, open(output_path, "wb") as dst: - while True: - buf = src.read(65536) - if not buf: - break - dst.write(buf) + target_soabi = soabi_override or _soabi_for(target_os, arch) + host_soabi = _host_soabi() - print(f"[rebuild] Copied original wheel to: {output_path}") + plat_tag = f"{target_os}_{arch}" + wheel_name = f"{pkg_name}-{version}-cp312-cp312-{plat_tag}.whl" + wheel_path = os.path.join(output_dir, wheel_name) - stub_path = output_path.replace(".whl", f"_{target_os}_{arch}_stub.txt") - with open(stub_path, "w") as f: - f.write(f"Native rebuild stub for {basename} ({target_os}/{arch})\n") - f.write("Replace this file with the actual rebuilt harmonyos wheel.\n") + dist_info = f"{pkg_name}-{version}.dist-info" + records = [] - print(f"[rebuild] Stub marker: {stub_path}") + def add_entry(zf, arcname, data_or_path): + if isinstance(data_or_path, str) and os.path.isfile(data_or_path): + with open(data_or_path, "rb") as f: + data = f.read() + elif isinstance(data_or_path, str): + data = data_or_path.encode() + else: + data = data_or_path + zf.writestr(arcname, data) + digest = hashlib.sha256(data).digest() + b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode() + records.append(f"{arcname},sha256={b64},{len(data)}") + + def add_symlink(zf, arcname, target): + info = zipfile.ZipInfo(arcname) + info.external_attr = 0o120777 << 16 + zf.writestr(info, target) + records.append(f"{arcname},,0") + + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + # ── dist-info metadata ────────────────────────── + add_entry(zf, f"{dist_info}/METADATA", ( + f"Metadata-Version: 2.1\n" + f"Name: {pkg_name}\n" + f"Version: {version}\n" + f"Summary: {pkg_name} \u2014 EcoArk HarmonyOS experimental cross-build\n" + f"Description: EcoArk HarmonyOS experimental cross-build for {target_os}/{arch}\n" + f" Built from source: {source_dir}\n" + f" Cross-compiled with: OHOS NDK clang 15.0.4\n" + f" This is an EXPERIMENTAL build \u2014 not yet formally released.\n" + f"License: See package license\n" + f"Requires-Python: >=3.9\n" + )) + add_entry(zf, f"{dist_info}/WHEEL", ( + f"Wheel-Version: 1.0\n" + f"Generator: ecoark-pipeline 0.1\n" + f"Root-Is-Purelib: false\n" + f"Tag: cp312-cp312-{plat_tag}\n" + )) + add_entry(zf, f"{dist_info}/top_level.txt", f"{pkg_name}\n") + add_entry(zf, f"{dist_info}/INSTALLER", "ecoark-pipeline\n") + add_entry(zf, f"{dist_info}/entry_points.txt", + "[console_scripts]\nf2py = numpy.f2py.f2py2e:main\n") + + # ── Copy Python source files ───────────────────── + pkg_src = os.path.join(source_dir, pkg_name) + if os.path.isdir(pkg_src): + for dirpath, dirnames, filenames in os.walk(pkg_src): + rel = os.path.relpath(dirpath, source_dir).replace("\\", "/") + # Prune __pycache__ + if "__pycache__" in dirnames: + dirnames.remove("__pycache__") + for fn in filenames: + src = os.path.join(dirpath, fn) + arcname = f"{rel}/{fn}" + if _should_include_source(fn, arcname): + add_entry(zf, arcname, src) + + # Copy top-level .py files too (e.g., _config.py) + for fn in os.listdir(source_dir): + if fn.endswith(".py"): + src = os.path.join(source_dir, fn) + if os.path.isfile(src): + add_entry(zf, fn, src) + + # ── Copy compiled .so files (rename SOABI) ────── + build_pkg = os.path.join(build_dir, pkg_name) + so_count = 0 + if os.path.isdir(build_pkg): + for dirpath, dirnames, filenames in os.walk(build_pkg): + rel = os.path.relpath(dirpath, build_dir).replace("\\", "/") + for fn in filenames: + if not fn.endswith(".so"): + continue + src = os.path.join(dirpath, fn) + new_fn = fn.replace(host_soabi, target_soabi) + arcname = f"{rel}/{new_fn}" + add_entry(zf, arcname, src) + so_count += 1 + + # ── Also check for .so files in sub-packages directly + # in the build dir (some builds put them differently) + for dirpath, dirnames, filenames in os.walk(build_dir): + rel = os.path.relpath(dirpath, build_dir).replace("\\", "/") + for fn in filenames: + if not fn.endswith(".so"): + continue + if not fn.startswith(pkg_name) and rel not in (".", ""): + continue + src = os.path.join(dirpath, fn) + new_fn = fn.replace(host_soabi, target_soabi) + arcname = f"{rel}/{new_fn}" + # Only add if not already added + if arcname not in [r.split(",")[0] for r in records]: + add_entry(zf, arcname, src) + so_count += 1 + + if so_count == 0: + print(f"[rebuild] WARNING: no .so files found in build dir {build_dir}") + + # ── RECORD (final) ───────────────────────────── + record_content = "\n".join(records) + "\n" + add_entry(zf, f"{dist_info}/RECORD", record_content) + + print(f"[rebuild] Wheel: {wheel_path}") + print(f"[rebuild] Size : {os.path.getsize(wheel_path)} bytes") + print(f"[rebuild] .so : {so_count} files (SOABI: {host_soabi} -> {target_soabi})") + return wheel_path + + +def build_numpy_wheel(version="1.26.4", output_dir=None, rebuild=False): + """Full numpy cross-compilation + wheel assembly pipeline. + + Steps: + 1. Download + extract numpy source sdist from PyPI + 2. Cross-compile with vendored meson + OHOS cross file + 3. Assemble wheel from Python source + cross-compiled .so files + """ + if not _check_toolchain(): + return None + + out_dir = output_dir or OUTPUT_DIR + sources = _ensure_dirs(SOURCES_DIR) + + # Download + extract + tarball = _download_sdist("numpy", version, sources) + src_dir = _extract_tarball(tarball, sources) + + build_dir = os.path.join(sources, f"numpy-{version}-build") + if os.path.isdir(build_dir) and not rebuild: + print(f"[rebuild] Build dir already exists: {build_dir}") + print("[rebuild] Set rebuild=True or remove the dir to rebuild from scratch") + else: + ok, build_dir = cross_compile_numpy(src_dir, build_dir) + if not ok: + print("[rebuild] Cross-compilation FAILED") + return None + + # Assemble wheel + wheel_path = assemble_wheel(src_dir, build_dir, out_dir, + pkg_name="numpy", version=version, + target_os="harmonyos", arch="arm64") + return wheel_path + + +def rebuild_from_source(source_dir, pkg_name, output_dir, version="0.0.0", + target_os="harmonyos", arch="arm64"): + """Generic entry point: rebuild a source package for HarmonyOS. + + For now, dispatches to package-specific builders. + Falls back to stub wheel generation for unknown packages. + """ + if pkg_name == "numpy": + return build_numpy_wheel(version=version, output_dir=output_dir) + return _stub_wheel(source_dir, pkg_name, output_dir, version, target_os, arch) + + +def rebuild_native_wheel(wheel_path: str, output_dir: str, + target_os: str = "harmonyos", arch: str = "arm64") -> str: + """Entry point for wheel-to-wheel rebuild (not yet implemented).""" + print(f"[rebuild] Native wheel rebuild from wheel not yet supported: {wheel_path}") + print("[rebuild] Use --manifest or rebuild_from_source() instead.") + basename = os.path.basename(wheel_path) + output_path = os.path.join(output_dir, basename) + os.makedirs(output_dir, exist_ok=True) + shutil.copy2(wheel_path, output_path) return output_path +def _stub_wheel(source_dir, pkg_name, output_dir, version, target_os, arch): + """Generate a stub wheel with metadata only.""" + os.makedirs(output_dir, exist_ok=True) + wheel_name = f"{pkg_name}-{version}-cp312-cp312-{target_os}_{arch}.whl" + wheel_path = os.path.join(output_dir, wheel_name) + dist_info = f"{pkg_name}-{version}.dist-info" + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr(f"{dist_info}/METADATA", + f"Metadata-Version: 2.1\nName: {pkg_name}\nVersion: {version}\n" + f"Summary: EcoArk HarmonyOS rebuild (stub)\n") + zf.writestr(f"{dist_info}/WHEEL", + f"Wheel-Version: 1.0\nGenerator: ecoark-pipeline\n" + f"Root-Is-Purelib: false\nTag: cp312-cp312-{target_os}_{arch}\n") + zf.writestr(f"{dist_info}/RECORD", "") + zf.writestr(f"{pkg_name}/__init__.py", + f"def hello(): return 'built by EcoArk pipeline (stub)'\n") + zf.writestr(f"{pkg_name}/_ecoark_stub.txt", + f"Package: {pkg_name}\nSource: {source_dir}\n" + f"Target: {target_os}/{arch}\nStub wheel (replace with real build).\n") + print(f"[rebuild] Stub wheel: {wheel_path}") + return wheel_path + + if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python -m pipeline.rebuild [output_dir]") - sys.exit(1) - rebuild_native_wheel(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "output") + import sys + if len(sys.argv) > 1 and sys.argv[1] == "numpy": + build_numpy_wheel(version=sys.argv[2] if len(sys.argv) > 2 else "1.26.4") + else: + print("Usage: python -m pipeline.rebuild numpy [version]") + print(" python -m pipeline.rebuild numpy 1.26.4") diff --git a/recipes/numpy_integration_manifest.json b/recipes/numpy_integration_manifest.json new file mode 100644 index 0000000..560f1fe --- /dev/null +++ b/recipes/numpy_integration_manifest.json @@ -0,0 +1,31 @@ +{ + "pkg": "numpy", + "version": "1.26.4", + "description": "Integration staging recipe for numpy native .so modules into EcoArk pythonrunner main-so linking", + "staging": { + "source_wheel": "output/wheels/numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl", + "output_dir": "output/integration/numpy" + }, + "modules": [ + {"so": "numpy/core/_multiarray_umath.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_multiarray_umath.so"}, + {"so": "numpy/core/_multiarray_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_multiarray_tests.so"}, + {"so": "numpy/core/_simd.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_simd.so"}, + {"so": "numpy/core/_operand_flag_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_operand_flag_tests.so"}, + {"so": "numpy/core/_struct_ufunc_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_struct_ufunc_tests.so"}, + {"so": "numpy/core/_rational_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_rational_tests.so"}, + {"so": "numpy/core/_umath_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_umath_tests.so"}, + {"so": "numpy/fft/_pocketfft_internal.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/fft/_pocketfft_internal.so"}, + {"so": "numpy/linalg/_umath_linalg.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/linalg/_umath_linalg.so"}, + {"so": "numpy/linalg/lapack_lite.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/linalg/lapack_lite.so"}, + {"so": "numpy/random/_generator.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_generator.so"}, + {"so": "numpy/random/mtrand.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/mtrand.so"}, + {"so": "numpy/random/_common.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_common.so"}, + {"so": "numpy/random/_bounded_integers.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_bounded_integers.so"}, + {"so": "numpy/random/bit_generator.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/bit_generator.so"}, + {"so": "numpy/random/_sfc64.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_sfc64.so"}, + {"so": "numpy/random/_pcg64.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_pcg64.so"}, + {"so": "numpy/random/_philox.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_philox.so"}, + {"so": "numpy/random/_mt19937.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_mt19937.so"} + ], + "ecoark_target_hint": "Copy each .so into EcoArk pythonrunner's native module tree (e.g. lib/python3.12/site-packages/numpy/) so that import numpy works with main-so linking." +} diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/stage_integration.py b/scripts/stage_integration.py new file mode 100644 index 0000000..68ab759 --- /dev/null +++ b/scripts/stage_integration.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Stage built native .so files for EcoArk pythonrunner main-so integration. + +Extracts .so files from an existing HarmonyOS-cross-compiled wheel and copies +them into an integration output directory with a README describing where each +module should be placed in the EcoArk pythonrunner environment. +""" +import argparse +import json +import os +import shutil +import sys +import zipfile + + +def stage_integration(wheel_path, output_dir, recipe=None): + os.makedirs(output_dir, exist_ok=True) + + if not os.path.isfile(wheel_path): + print(f"[stage] ERROR: wheel not found: {wheel_path}", file=sys.stderr) + return 1 + + staged = [] + with zipfile.ZipFile(wheel_path, "r") as z: + for name in z.namelist(): + if not name.endswith(".so"): + continue + dest = os.path.join(output_dir, name) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with z.open(name) as src, open(dest, "wb") as dst: + shutil.copyfileobj(src, dst) + staged.append(name) + print(f"[stage] {name}") + + readme_path = os.path.join(output_dir, "README.md") + with open(readme_path, "w") as f: + f.write(f"# Integration Staging for EcoArk pythonrunner main-so linking\n\n") + f.write(f"Source wheel: `{wheel_path}`\n\n") + f.write(f"## Staged .so files ({len(staged)} total)\n\n") + for s in staged: + f.write(f"- `{s}`\n") + f.write(f"\n## EcoArk target hint\n\n") + f.write("Copy each .so into EcoArk pythonrunner's native module tree\n") + f.write("under `lib/python3.12/site-packages/` preserving subdirectory\n") + f.write("structure so that `import numpy` resolves via main-so linking.\n") + so_list_path = os.path.join(output_dir, "integration_manifest.json") + manifest = { + "pkg": "numpy", + "staged_count": len(staged), + "ecoark_target": "lib/python3.12/site-packages/", + "files": staged, + } + if recipe and os.path.isfile(recipe): + with open(recipe) as rf: + manifest["recipe"] = json.load(rf) + with open(so_list_path, "w") as f: + json.dump(manifest, f, indent=2) + + print(f"\n[stage] Staged {len(staged)} .so files to {output_dir}") + print(f"[stage] Manifest: {so_list_path}") + print(f"[stage] README: {readme_path}") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Stage native .so files from a built HarmonyOS wheel for EcoArk integration" + ) + parser.add_argument("wheel", nargs="?", help="Path to built HarmonyOS .whl file") + parser.add_argument("--output-dir", "-o", default="output/integration", + help="Integration output directory (default: output/integration)") + parser.add_argument("--recipe", help="Path to integration recipe JSON (optional)") + parser.add_argument("--list-recipes", action="store_true", + help="List available recipes and exit") + args = parser.parse_args() + + if args.list_recipes: + recipes_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "recipes") + print("Available integration recipes:") + if os.path.isdir(recipes_dir): + for fn in sorted(os.listdir(recipes_dir)): + if fn.endswith(".json"): + rp = os.path.join(recipes_dir, fn) + with open(rp) as f: + d = json.load(f) + print(f" {fn} ({d.get('pkg','?')} {d.get('version','?')})") + else: + print(" (no recipes directory)") + return 0 + + if not args.wheel: + # Default: look for numpy wheel in output/wheels + default = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "output", "wheels", "numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl" + ) + if os.path.isfile(default): + args.wheel = default + print(f"[stage] Using default wheel: {default}") + else: + parser.error("no wheel specified and default numpy wheel not found") + + return stage_integration(args.wheel, os.path.abspath(args.output_dir), recipe=args.recipe) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/upload_artifacts.py b/scripts/upload_artifacts.py new file mode 100644 index 0000000..14ec080 --- /dev/null +++ b/scripts/upload_artifacts.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +""" +Upload build outputs as Gitea Actions artifacts using the Actions Runtime API. + +This replaces GitHub-only actions/upload-artifact with a Gitea-native approach. +Uses ACTIONS_RUNTIME_URL and ACTIONS_RUNTIME_TOKEN provided by Gitea's +act_runner to upload artifacts directly to the pipeline service. + +Usage: + python3 scripts/upload_artifacts.py [path_glob2 ...] + +Environment (set by Gitea act_runner >= 1.21): + ACTIONS_RUNTIME_URL — Pipeline service base URL (e.g. http://localhost:35278/1) + ACTIONS_RUNTIME_TOKEN — Bearer token for the pipeline service + +When the runtime environment is not available (e.g. local testing), +artifacts are saved to ./artifacts/.zip as a fallback. +""" +import glob +import io +import os +import sys +import urllib.error +import urllib.request +import zipfile + +_RUNTIME_UPLOADED = False + + +def _upload_via_runtime(name, payload, runtime_url, runtime_token): + url = "{}/_apis/pipeline/artifacts/{}".format(runtime_url.rstrip("/"), name) + headers = { + "Authorization": "Bearer " + runtime_token, + "Content-Type": "application/octet-stream", + "Content-Range": "bytes 0-{}/{}".format(len(payload) - 1, len(payload)), + "x-tfs-filelength": str(len(payload)), + } + req = urllib.request.Request(url, data=payload, headers=headers, method="PUT") + try: + with urllib.request.urlopen(req) as resp: + msg = "[upload-artifact] Uploaded '{}' ({} bytes) to Gitea runtime — HTTP {}" + print(msg.format(name, len(payload), resp.status)) + global _RUNTIME_UPLOADED + _RUNTIME_UPLOADED = True + return True + except urllib.error.HTTPError as e: + body = e.read().decode()[:300] if e.fp else "" + print("[upload-artifact] Runtime upload failed: HTTP {} — {}".format(e.code, body)) + return False + + +def _save_locally(name, payload): + dest = os.path.join("artifacts", "{}.zip".format(name)) + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + with open(dest, "wb") as f: + f.write(payload) + print("[upload-artifact] Saved artifact to {} ({} bytes)".format(dest, len(payload))) + + +def main(): + if len(sys.argv) < 3: + print("Usage: {} [path_glob ...]".format(sys.argv[0])) + sys.exit(1) + + artifact_name = sys.argv[1] + patterns = sys.argv[2:] + + files = [] + for pattern in patterns: + matched = glob.glob(pattern, recursive=True) + files.extend(matched) + files = sorted(set(f for f in files if os.path.isfile(f))) + + if not files: + print("[upload-artifact] No files found for pattern(s): {}".format(patterns)) + return + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for f in files: + zf.write(f, os.path.relpath(f)) + payload = buf.getvalue() + + runtime_url = os.environ.get("ACTIONS_RUNTIME_URL", "").rstrip("/") + runtime_token = os.environ.get("ACTIONS_RUNTIME_TOKEN", "") + + if runtime_url and runtime_token: + ok = _upload_via_runtime(artifact_name, payload, runtime_url, runtime_token) + if not ok: + _save_locally(artifact_name, payload) + else: + print("[upload-artifact] ACTIONS_RUNTIME_URL/TOKEN not available — saving locally.") + _save_locally(artifact_name, payload) + + print("[upload-artifact] Done — {} file(s) in '{}'.".format(len(files), artifact_name)) + + +if __name__ == "__main__": + main() diff --git a/scripts/upload_release_assets.py b/scripts/upload_release_assets.py new file mode 100644 index 0000000..6541f16 --- /dev/null +++ b/scripts/upload_release_assets.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +Upload wheel / zip assets to a Gitea Release. + +Environment variables (set by Gitea runner or user): + ENABLE_RELEASE_UPLOAD — set to "true" to activate (default: skip) + GITEA_TOKEN — Gitea access token with `repo` scope + RELEASE_TOKEN — fallback if GITEA_TOKEN is unset + GITHUB_SERVER_URL — Gitea server URL (runner-provided) + GITHUB_REPOSITORY — owner/repo (runner-provided) + GITHUB_REF_NAME — tag name when pushing a tag (runner-provided) + GITHUB_SHA — commit SHA (runner-provided) + +Usage: + python3 scripts/upload_release_assets.py +""" +import glob +import json +import os +import secrets +import sys +import urllib.error +import urllib.request + + +def env(key, default=""): + return os.environ.get(key, default) + + +def _multipart_formdata(field, filename, filepath): + boundary = "----GiteaReleaseBoundary{:x}".format(secrets.randbits(64)) + with open(filepath, "rb") as f: + payload = f.read() + body = ( + "--{}\r\n" + 'Content-Disposition: form-data; name="{}"; filename="{}"\r\n' + "Content-Type: application/octet-stream\r\n\r\n" + ).format(boundary, field, filename).encode() + payload + ( + "\r\n--{}--\r\n".format(boundary) + ).encode() + return boundary, body + + +def _api(method, url, headers, data=None, filepath=None): + req_headers = dict(headers) + body = None + if filepath: + boundary, body = _multipart_formdata("attachment", os.path.basename(filepath), filepath) + req_headers["Content-Type"] = "multipart/form-data; boundary={}".format(boundary) + elif data is not None: + body = json.dumps(data).encode() + req_headers.setdefault("Content-Type", "application/json") + req = urllib.request.Request(url, data=body, headers=req_headers, method=method) + try: + with urllib.request.urlopen(req) as resp: + return resp.status, json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + snippet = e.read().decode()[:200] if e.fp else "" + print(" [release] HTTP {} on {} {}: {}".format(e.code, method, url, snippet)) + return e.code, None + + +def main(): + # ── Guard ── + if env("ENABLE_RELEASE_UPLOAD").lower() not in ("true", "1", "yes"): + print("[release] ENABLE_RELEASE_UPLOAD not active — skipping.") + return + + token = env("GITEA_TOKEN") or env("RELEASE_TOKEN") + if not token: + print("[release] No token (GITEA_TOKEN / RELEASE_TOKEN) — skipping.") + return + + server_url = env("GITHUB_SERVER_URL").rstrip("/") + repo = env("GITHUB_REPOSITORY") + ref_name = env("GITHUB_REF_NAME") + sha = env("GITHUB_SHA") + + if not repo: + print("[release] GITHUB_REPOSITORY unset — skipping.") + return + + # ── Scan assets ── + assets_dir = sys.argv[1] if len(sys.argv) > 1 else "output" + whl_files = sorted( + glob.glob(os.path.join(assets_dir, "*.whl")) + + glob.glob(os.path.join(assets_dir, "wheels", "*.whl")) + ) + zip_files = sorted(glob.glob(os.path.join(assets_dir, "*.zip"))) + all_files = whl_files + zip_files + + if not all_files: + print("[release] No .whl or .zip found under {}/ — nothing to upload.".format(assets_dir)) + return + + tag = ref_name or ("build-{}".format(sha[:7]) if sha else "latest") + + # ── Gitea API ── + api_base = "{}/api/v1/repos/{}".format(server_url, repo) + headers = {"Authorization": "token {}".format(token), "Accept": "application/json"} + + # 1) Get or create release + print("[release] Target tag: {}".format(tag)) + status, release = _api("GET", "{}/releases/tags/{}".format(api_base, tag), headers) + if status == 200 and release: + release_id = release["id"] + print("[release] Found existing release #{} ({})".format(release_id, tag)) + else: + print("[release] Creating release for tag {} ...".format(tag)) + status, release = _api("POST", "{}/releases".format(api_base), headers, { + "tag_name": tag, + "target_commitish": sha, + "name": "EcoArk Wheels {}".format(tag), + "body": "Automated wheel build from commit {}.".format(sha[:7] if sha else "unknown"), + "draft": False, + "prerelease": True, + }) + if status not in (200, 201): + print("[release] Failed to create release — skipping asset upload.") + return + release_id = release["id"] + print("[release] Created release #{} ({})".format(release_id, tag)) + + # 2) Upload each asset as an attachment + for fpath in all_files: + fname = os.path.basename(fpath) + print("[release] Uploading {} ...".format(fname)) + _api("POST", "{}/releases/{}/assets".format(api_base, release_id), headers, filepath=fpath) + + print('[release] Done — {} asset(s) uploaded to "{}".'.format(len(all_files), tag)) + + +if __name__ == "__main__": + main()