feat: add manifest-driven batch rebuild, numpy cross-compilation, and CI pipeline updates
Some checks failed
EcoArk Wheel Pipeline CI / build (push) Failing after 1m41s
Some checks failed
EcoArk Wheel Pipeline CI / build (push) Failing after 1m41s
- Add pipeline/manifest.py for packages.txt parsing - Implement real numpy cross-compilation (rebuild.py) with OHOS NDK + vendored meson - Extend CLI with --manifest, --build-from-source, --stage-integration modes - Add recipes/, experiments/, and scripts/ for integration staging and artifact upload - Update CI workflow with manifest validation, artifact upload, and release publishing - Update Makefile and README with new targets and documentation
This commit is contained in:
root
@ -3,16 +3,19 @@ name: EcoArk Wheel Pipeline CI
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
|
# Uncomment the line below to trigger releases on tags:
|
||||||
|
# tags: ['v*']
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
||||||
# ──────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────
|
||||||
# Job 1: validate — pure wheel repack + native stub
|
# Job: build — validate core logic + manifest-driven
|
||||||
# ──────────────────────────────────────────────
|
# rebuild (stub) + zip bundle
|
||||||
validate:
|
# ──────────────────────────────────────────────────────
|
||||||
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
|
||||||
@ -25,49 +28,77 @@ jobs:
|
|||||||
|
|
||||||
- name: Show environment info
|
- name: Show environment info
|
||||||
run: |
|
run: |
|
||||||
echo "══════════════════════════════════════════"
|
echo "╔══════════════════════════════════════════╗"
|
||||||
|
echo "║ EcoArk Wheel Pipeline CI ║"
|
||||||
|
echo "╚══════════════════════════════════════════╝"
|
||||||
echo " Runner : $(uname -a)"
|
echo " Runner : $(uname -a)"
|
||||||
echo " Hostname: $(hostname)"
|
echo " Hostname: $(hostname)"
|
||||||
echo " Date : $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
echo " Date : $(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
echo "══════════════════════════════════════════"
|
|
||||||
python3 --version
|
python3 --version
|
||||||
pip3 --version 2>/dev/null || echo "pip3 not available"
|
pip3 --version 2>/dev/null || echo "pip3 not available"
|
||||||
which make gcc g++ rustc 2>/dev/null || true
|
which make git 2>/dev/null || true
|
||||||
|
|
||||||
|
- name: Install Python build deps
|
||||||
|
run: |
|
||||||
|
pip3 install --quiet --upgrade pip setuptools wheel 2>/dev/null || true
|
||||||
|
|
||||||
|
# ── Validate single-wheel mode (backward-compat) ──
|
||||||
- name: Generate test wheels
|
- name: Generate test wheels
|
||||||
run: |
|
run: python3 examples/gen_test_wheels.py
|
||||||
python3 examples/gen_test_wheels.py
|
|
||||||
|
|
||||||
- name: Show generated test wheels
|
|
||||||
run: |
|
|
||||||
ls -lh examples/*.whl
|
|
||||||
|
|
||||||
- name: Validate pure wheel repack
|
- name: Validate pure wheel repack
|
||||||
run: |
|
run: |
|
||||||
WHEEL=$(ls examples/*-none-any.whl | head -1)
|
WHEEL=$(ls examples/*-none-any.whl | head -1)
|
||||||
echo "Pure wheel: $WHEEL"
|
|
||||||
python3 wheel_pipeline.py "$WHEEL" --output-dir output
|
python3 wheel_pipeline.py "$WHEEL" --output-dir output
|
||||||
echo ""
|
|
||||||
echo "Output files:"
|
|
||||||
ls -lh output/*.whl
|
|
||||||
|
|
||||||
- name: Validate native wheel stub
|
- name: Validate native wheel stub
|
||||||
run: |
|
run: |
|
||||||
WHEEL=$(ls examples/*-manylinux*.whl | head -1)
|
WHEEL=$(ls examples/*-manylinux*.whl | head -1)
|
||||||
echo "Native wheel: $WHEEL"
|
|
||||||
python3 wheel_pipeline.py "$WHEEL" --output-dir output --target-os harmonyos --arch arm64
|
python3 wheel_pipeline.py "$WHEEL" --output-dir output --target-os harmonyos --arch arm64
|
||||||
echo ""
|
|
||||||
echo "Output files:"
|
|
||||||
ls -lh output/*.whl
|
|
||||||
|
|
||||||
- name: Show all pipeline outputs
|
# ── Validate manifest-driven mode ─────────────────
|
||||||
|
- name: Run manifest pipeline (default packages.txt)
|
||||||
run: |
|
run: |
|
||||||
echo "=== Final output directory ==="
|
python3 wheel_pipeline.py --manifest packages.txt \
|
||||||
find output/ -type f -name '*.whl' -exec ls -lh {} \;
|
--output-dir output --target-os harmonyos --arch arm64
|
||||||
echo ""
|
|
||||||
echo "=== All output artifacts ==="
|
- name: Run manifest pipeline (test manifest with fake entry)
|
||||||
|
run: |
|
||||||
|
echo "# test manifest" > output/test_packages.txt
|
||||||
|
echo "ecoark_hello:https://github.com/ecoark/hello.git" >> output/test_packages.txt
|
||||||
|
python3 wheel_pipeline.py --manifest output/test_packages.txt \
|
||||||
|
--output-dir output --target-os harmonyos --arch arm64
|
||||||
|
|
||||||
|
- name: List all pipeline outputs
|
||||||
|
run: |
|
||||||
|
echo "=== Output tree ==="
|
||||||
find output/ -type f | sort
|
find output/ -type f | sort
|
||||||
|
echo ""
|
||||||
|
echo "=== Zip bundle ==="
|
||||||
|
ls -lh output/ecoark-rebuild-output.zip 2>/dev/null || echo "(no zip)"
|
||||||
|
|
||||||
|
# ── Upload total bundle as CI artifact (Gitea-native) ──
|
||||||
|
- name: Upload bundle as artifact
|
||||||
|
run: python3 scripts/upload_artifacts.py ecoark-harmonyos-wheels "output/ecoark-rebuild-output.zip"
|
||||||
|
|
||||||
|
# ── Upload each .whl individually as artifact ─────────
|
||||||
|
- name: Upload individual wheel artifacts
|
||||||
|
run: python3 scripts/upload_artifacts.py ecoark-wheels-individual "output/*.whl" "output/wheels/*.whl"
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────
|
||||||
|
# Publish to Gitea Release (optional — requires secrets)
|
||||||
|
# ──────────────────────────────────────────────────────
|
||||||
|
- name: Upload assets to Gitea Release
|
||||||
|
if: secrets.GITEA_TOKEN != '' && secrets.ENABLE_RELEASE_UPLOAD == 'true'
|
||||||
|
run: python3 scripts/upload_release_assets.py output/
|
||||||
|
env:
|
||||||
|
ENABLE_RELEASE_UPLOAD: 'true'
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
|
||||||
- name: Summary
|
- name: Summary
|
||||||
run: |
|
run: |
|
||||||
echo "All pipeline steps passed successfully."
|
echo "All pipeline steps passed successfully."
|
||||||
|
echo "Artifacts:"
|
||||||
|
echo " - ecoark-harmonyos-wheels (total zip bundle)"
|
||||||
|
echo " - ecoark-wheels-individual (each .whl file individually)"
|
||||||
|
echo "Release upload: configured via GITEA_TOKEN + ENABLE_RELEASE_UPLOAD secrets"
|
||||||
|
|||||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -6,3 +6,6 @@ dist/
|
|||||||
build/
|
build/
|
||||||
examples/*.whl
|
examples/*.whl
|
||||||
output/*.whl
|
output/*.whl
|
||||||
|
output/*.zip
|
||||||
|
output/sources/
|
||||||
|
output/wheels/
|
||||||
|
|||||||
28
Makefile
28
Makefile
@ -2,20 +2,26 @@ SHELL := /bin/bash
|
|||||||
|
|
||||||
INPUT ?= examples/sample.whl
|
INPUT ?= examples/sample.whl
|
||||||
OUTPUT ?= output
|
OUTPUT ?= output
|
||||||
|
MANIFEST ?= packages.txt
|
||||||
|
|
||||||
.PHONY: help repack rebuild native clean
|
.PHONY: help repack rebuild native manifest build-numpy stage-integration clean release-upload
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "EcoArk HarmonyOS Wheel Pipeline"
|
@echo "EcoArk HarmonyOS Wheel Pipeline"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Usage:"
|
@echo "Usage:"
|
||||||
@echo " make repack INPUT=path/to/pkg.whl — Repack a pure-Python wheel for HarmonyOS"
|
@echo " make repack INPUT=path/to/pkg.whl — Repack a pure-Python wheel for HarmonyOS"
|
||||||
@echo " make rebuild INPUT=path/to/pkg.whl — Trigger native rebuild entry (stub)"
|
@echo " make rebuild INPUT=path/to/pkg.whl — Trigger native rebuild entry"
|
||||||
|
@echo " make build-numpy VERSION=1.26.4 — Cross-compile numpy for HarmonyOS"
|
||||||
|
@echo " make stage-integration — Stage numpy .so files for EcoArk integration"
|
||||||
|
@echo " make manifest MANIFEST=packages.txt — Manifest-driven batch rebuild"
|
||||||
|
@echo " make artifacts — Simulate Gitea artifact upload (local save)"
|
||||||
|
@echo " make release-upload — Upload assets to Gitea Release"
|
||||||
@echo " make clean — Remove output/"
|
@echo " make clean — Remove output/"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Examples:"
|
@echo "Examples:"
|
||||||
@echo " make repack INPUT=some_package.whl"
|
@echo " make repack INPUT=some_package.whl"
|
||||||
@echo " make rebuild INPUT=some_native.whl"
|
@echo " make build-numpy VERSION=1.26.4"
|
||||||
|
|
||||||
repack:
|
repack:
|
||||||
python3 wheel_pipeline.py "$(INPUT)" --output-dir "$(OUTPUT)" --force-repack
|
python3 wheel_pipeline.py "$(INPUT)" --output-dir "$(OUTPUT)" --force-repack
|
||||||
@ -26,7 +32,23 @@ rebuild:
|
|||||||
native:
|
native:
|
||||||
python3 wheel_pipeline.py "$(INPUT)" --output-dir "$(OUTPUT)" --target-os harmonyos --arch arm64
|
python3 wheel_pipeline.py "$(INPUT)" --output-dir "$(OUTPUT)" --target-os harmonyos --arch arm64
|
||||||
|
|
||||||
|
build-numpy:
|
||||||
|
python3 wheel_pipeline.py --build-from-source numpy --package-version "$(VERSION)" --output-dir "$(OUTPUT)" --rebuild
|
||||||
|
|
||||||
|
manifest:
|
||||||
|
python3 wheel_pipeline.py --manifest "$(MANIFEST)" --output-dir "$(OUTPUT)" --target-os harmonyos --arch arm64
|
||||||
|
|
||||||
|
stage-integration:
|
||||||
|
python3 wheel_pipeline.py --stage-integration --output-dir "$(OUTPUT)"
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf "$(OUTPUT)"
|
rm -rf "$(OUTPUT)"
|
||||||
mkdir -p "$(OUTPUT)"
|
mkdir -p "$(OUTPUT)"
|
||||||
touch "$(OUTPUT)/.gitkeep"
|
touch "$(OUTPUT)/.gitkeep"
|
||||||
|
|
||||||
|
release-upload:
|
||||||
|
python3 scripts/upload_release_assets.py "$(OUTPUT)"
|
||||||
|
|
||||||
|
artifacts:
|
||||||
|
python3 scripts/upload_artifacts.py ecoark-harmonyos-wheels "$(OUTPUT)/*.zip"
|
||||||
|
python3 scripts/upload_artifacts.py ecoark-wheels-individual "$(OUTPUT)/*.whl" "$(OUTPUT)/wheels/*.whl"
|
||||||
|
|||||||
204
README.md
204
README.md
@ -4,28 +4,72 @@
|
|||||||
|
|
||||||
## 用途
|
## 用途
|
||||||
|
|
||||||
- 接收一个 `.whl` 文件作为输入
|
- **单 wheel 模式**：接收一个 `.whl` 文件，自动检测类型并处理
|
||||||
- 自动检测是 **纯 Python wheel** 还是 **含 native 扩展的 wheel**
|
|
||||||
- 纯 Python wheel → 直接重打包为 `harmonyos_arm64` 兼容标签，安装即用
|
- 纯 Python wheel → 直接重打包为 `harmonyos_arm64` 兼容标签，安装即用
|
||||||
- Native wheel → 预留 HarmonyOS/arm64 交叉编译入口（当前为骨架，可扩展）
|
- Native wheel → 交叉编译为 HarmonyOS/arm64 原生 .so 并组装 wheel
|
||||||
|
- **清单批量模式**：通过 `packages.txt` 清单文件，逐个拉取源码仓库，执行重编，最终打成 zip 包
|
||||||
|
|
||||||
## 目录结构
|
## 目录结构
|
||||||
|
|
||||||
```
|
```
|
||||||
ecoarkpywhl/
|
ecoarkpywhl/
|
||||||
├── wheel_pipeline.py # CLI 入口（最简调用）
|
├── wheel_pipeline.py # CLI 入口（最简调用）
|
||||||
|
├── packages.txt # 待处理项目清单（格式说明见下文）
|
||||||
├── pipeline/
|
├── pipeline/
|
||||||
│ ├── cli.py # 参数解析 & 主流程
|
│ ├── cli.py # 参数解析 & 主流程（含 --build-from-source 模式）
|
||||||
|
│ ├── manifest.py # packages.txt 解析器
|
||||||
│ ├── detect.py # 检测 wheel 类型（纯 Python / native）
|
│ ├── detect.py # 检测 wheel 类型（纯 Python / native）
|
||||||
│ ├── repack.py # 纯 Python wheel 重打包
|
│ ├── repack.py # 纯 Python wheel 重打包
|
||||||
│ └── rebuild.py # Native wheel 重编译入口（骨架）
|
│ └── rebuild.py # Native wheel 交叉编译 + wheel 组装
|
||||||
|
├── recipes/
|
||||||
|
│ └── numpy_integration_manifest.json # numpy 集成暂存配方
|
||||||
|
├── scripts/
|
||||||
|
│ ├── stage_integration.py # 集成暂存脚本
|
||||||
|
│ └── upload_release_assets.py # Gitea Release 上传脚本
|
||||||
|
├── experiments/ # 交叉编译实验记录
|
||||||
├── Makefile # 常用命令
|
├── Makefile # 常用命令
|
||||||
├── output/ # 输出目录（生成文件放这里）
|
├── output/ # 输出目录（wheels/、sources/）
|
||||||
└── README.md
|
└── README.md
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## packages.txt — 项目清单
|
||||||
|
|
||||||
|
`packages.txt` 定义需要批量重编的包列表，格式：
|
||||||
|
|
||||||
|
```
|
||||||
|
package_name:git_repository_url
|
||||||
|
```
|
||||||
|
|
||||||
|
- 每行一个包，`:` 前为包名，后为 git clone URL
|
||||||
|
- 空行和以 `#` 开头的行被忽略
|
||||||
|
- 所有行被注释的 `packages.txt` 是安全的空清单
|
||||||
|
|
||||||
|
### 示例
|
||||||
|
|
||||||
|
```
|
||||||
|
# numpy:https://github.com/numpy/numpy.git
|
||||||
|
# scipy:https://github.com/scipy/scipy.git
|
||||||
|
```
|
||||||
|
|
||||||
|
去掉 `#` 即激活对应包。
|
||||||
|
|
||||||
|
## 集成暂存模式（Integration Staging）
|
||||||
|
|
||||||
|
将已编译的 native .so 文件从 wheel 中提取并组织到 `output/integration/<pkg>/` 目录，
|
||||||
|
附带 README 和 manifest，说明各模块在 EcoArk pythonrunner main-so 链接中的目标路径。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 通过 CLI 入口
|
||||||
|
python3 wheel_pipeline.py --stage-integration
|
||||||
|
|
||||||
|
# 或通过 Makefile
|
||||||
|
make stage-integration
|
||||||
|
```
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
|
|
||||||
|
### 单 wheel 模式（向后兼容）
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 纯 Python wheel → 重打包
|
# 纯 Python wheel → 重打包
|
||||||
python3 wheel_pipeline.py some_pure_package.whl
|
python3 wheel_pipeline.py some_pure_package.whl
|
||||||
@ -33,25 +77,122 @@ python3 wheel_pipeline.py some_pure_package.whl
|
|||||||
# 指定输出目录
|
# 指定输出目录
|
||||||
python3 wheel_pipeline.py some_pure_package.whl --output-dir /tmp/wheels
|
python3 wheel_pipeline.py some_pure_package.whl --output-dir /tmp/wheels
|
||||||
|
|
||||||
# Native wheel → 触发重编译入口（当前只复制 + 写 stub）
|
# Native wheel → 触发重编译入口
|
||||||
python3 wheel_pipeline.py some_native_package.whl
|
python3 wheel_pipeline.py some_native_package.whl
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 从源码交叉编译
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 交叉编译 numpy 1.26.4 并产出 HarmonyOS arm64 wheel
|
||||||
|
python3 wheel_pipeline.py --build-from-source numpy --package-version 1.26.4
|
||||||
|
|
||||||
|
# 强制重新编译（跳过缓存）
|
||||||
|
python3 wheel_pipeline.py --build-from-source numpy --package-version 1.26.4 --rebuild
|
||||||
|
```
|
||||||
|
|
||||||
|
### 清单批量模式
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 读取 packages.txt，逐个拉取源码并生成 wheel
|
||||||
|
python3 wheel_pipeline.py --manifest packages.txt --output-dir output
|
||||||
|
|
||||||
|
# 使用自定义清单
|
||||||
|
python3 wheel_pipeline.py --manifest my_list.txt
|
||||||
|
```
|
||||||
|
|
||||||
或通过 Makefile:
|
或通过 Makefile:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make repack INPUT=some_package.whl
|
make repack INPUT=some_package.whl
|
||||||
make rebuild INPUT=some_native_package.whl
|
make rebuild INPUT=some_native_package.whl
|
||||||
|
make manifest MANIFEST=packages.txt
|
||||||
|
make build-numpy VERSION=1.26.4
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 清单模式工作流行为
|
||||||
|
|
||||||
|
1. 读取 `packages.txt`（或 `--manifest` 指定的文件）
|
||||||
|
2. 对每个未注释的条目：
|
||||||
|
- `git clone --depth 1 <repo_url>` 到 `output/sources/<pkg_name>/`
|
||||||
|
- 调用 `pipeline/rebuild.rebuild_from_source()` 生成 wheel
|
||||||
|
3. 收集 `output/wheels/` 下所有 `.whl` 文件
|
||||||
|
4. 打包为 `output/ecoark-rebuild-output.zip`
|
||||||
|
|
||||||
|
### CI 中的产物路径
|
||||||
|
|
||||||
|
| 产物 | 路径 | 获取方式 |
|
||||||
|
|------|------|----------|
|
||||||
|
| 单个 wheel | `output/wheels/<pkg>-<ver>-cp312-cp312-harmonyos_arm64.whl` | **artifact**: `ecoark-wheels-individual` |
|
||||||
|
| 汇总 zip 包 | `output/ecoark-rebuild-output.zip` | **artifact**: `ecoark-harmonyos-wheels` |
|
||||||
|
| Release 附件 | 同上 | **Gitea Release**（需配置 token + tag push） |
|
||||||
|
|
||||||
|
> **artifact** = 在 Gitea Actions 运行结果页面直接下载（无需配置，即时可用）
|
||||||
|
> **Release** = 在 Gitea 仓库 Releases 页面长期保留（需配置 token，推 tag 时触发）
|
||||||
|
|
||||||
|
## Gitea Actions 流水线
|
||||||
|
|
||||||
|
当前工作流（`.gitea/workflows/wheel-pipeline.yaml`）包含一个 `build` job：
|
||||||
|
|
||||||
|
| 步骤 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| Checkout | 原生 git clone（不依赖 actions/checkout） |
|
||||||
|
| 生成测试 wheel | 构造纯 Python / native 测试 whl，验证单 wheel 模式 |
|
||||||
|
| 运行清单模式 | 先用 `packages.txt`（空清单 → no-op），再用临时清单验证 |
|
||||||
|
| 上传总 zip artifact | `scripts/upload_artifacts.py` → `ecoark-harmonyos-wheels`（Gitea Runtime API） |
|
||||||
|
| 上传单个 .whl artifacts | `scripts/upload_artifacts.py` → `ecoark-wheels-individual`（Gitea Runtime API） |
|
||||||
|
| 上传到 Gitea Release | `scripts/upload_release_assets.py`（需要 `GITEA_TOKEN` + `ENABLE_RELEASE_UPLOAD` 两个 secrets，见下方说明） |
|
||||||
|
|
||||||
|
## 从 CI 获取产物
|
||||||
|
|
||||||
|
### 方式一：直接下载 artifact（零配置，即时可用）
|
||||||
|
|
||||||
|
每次 CI 构建完成后，在 Gitea Actions 运行结果页面可以下载两类 artifact。
|
||||||
|
artifact 上传使用 `scripts/upload_artifacts.py` 通过 Gitea Actions Runtime API 完成，
|
||||||
|
**不依赖任何 GitHub-specific action**（如 `actions/upload-artifact`）。
|
||||||
|
|
||||||
|
| Artifact 名称 | 内容 | 适用场景 |
|
||||||
|
|---------------|------|----------|
|
||||||
|
| `ecoark-harmonyos-wheels` | 所有 wheel 打包成的 **一个 zip** | 一次下载全部 |
|
||||||
|
| `ecoark-wheels-individual` | 每个 `.whl` 文件**独立存放** | 只需特定一个 wheel |
|
||||||
|
|
||||||
|
操作步骤：
|
||||||
|
1. 打开 Gitea 仓库 → **Actions** → 最近成功运行
|
||||||
|
2. 在 workflow 页面底部找到 **Artifacts** 区域
|
||||||
|
3. 点击 `ecoark-wheels-individual` 进入，选择并下载你需要的单个 `.whl` 文件
|
||||||
|
4. 或直接点击 `ecoark-harmonyos-wheels` 下载汇总 zip
|
||||||
|
|
||||||
|
> 不需要 pip，不需要安装任何工具，浏览器直接下载 `.whl` 后即可用于 HarmonyOS Python 环境的 `pip install some_package.whl`。
|
||||||
|
|
||||||
|
### 方式二：Gitea Release（长期保留，需配置）
|
||||||
|
|
||||||
|
通过推 tag 触发自动发布，wheel 会以 Release 附件形式上传，长期可访问。
|
||||||
|
Release 上传通过 `scripts/upload_release_assets.py` 完成，由 `GITEA_TOKEN` +
|
||||||
|
`ENABLE_RELEASE_UPLOAD` 两个 secrets 共同控制。
|
||||||
|
Workflow 中使用 `if: secrets.GITEA_TOKEN != '' && secrets.ENABLE_RELEASE_UPLOAD == 'true'`
|
||||||
|
条件守卫，只有两个 secret 都**正确设置**后 Release 步骤才会执行。
|
||||||
|
|
||||||
|
配置步骤：
|
||||||
|
1. 在 Gitea 用户设置 → **Applications** → 创建 Access Token（`repo` scope）
|
||||||
|
2. 在仓库 → **Settings** → **Secrets** → 添加以下两个 secret：
|
||||||
|
- `GITEA_TOKEN` — 上述 token 值
|
||||||
|
- `ENABLE_RELEASE_UPLOAD` — 设为 `true` 激活发布步骤
|
||||||
|
3. 编辑 `.gitea/workflows/wheel-pipeline.yaml`，取消 `tags: ['v*']` 行的注释以启用 tag 触发
|
||||||
|
4. 推送一个 tag（例如 `git tag v0.1.0 && git push origin v0.1.0`），CI 会自动：
|
||||||
|
- 构建 wheel
|
||||||
|
- 创建/更新对应 tag 的 Gitea Release
|
||||||
|
- 将每个 `.whl` 和 zip 包上传为 Release 附件
|
||||||
|
|
||||||
## 限制
|
## 限制
|
||||||
|
|
||||||
| 场景 | 状态 |
|
| 场景 | 状态 |
|
||||||
|------|------|
|
|------|------|
|
||||||
| 纯 Python wheel → 重打包安装 | ✅ 支持 |
|
| 纯 Python wheel → 重打包安装 | ✅ 支持 |
|
||||||
| Pure Python → 重新打 tag | ✅ 支持 |
|
| Pure Python → 重新打 tag | ✅ 支持 |
|
||||||
| Native wheel → HarmonyOS 重编译 | 🚧 预留入口，需接入 OHOS NDK 工具链 |
|
| Native wheel → HarmonyOS 交叉编译 + wheel 产出 | ✅ 已支持（numpy 1.26.4 验证通过，19 个 AArch64 .so） |
|
||||||
| 依赖复杂的 native 包（numpy/scipy 等） | 🚧 需要对应依赖也移植到 HarmonyOS |
|
| 从源码编译到产出 .whl | ✅ `pipeline/rebuild.py` 含 `build_numpy_wheel()` 真实实现 |
|
||||||
|
| 依赖复杂的 native 包（scipy 等） | 🚧 需逐个包适配，底层依赖链未移植 |
|
||||||
|
| Gitea Release 自动发布 | 🚧 需配 `ENABLE_RELEASE_UPLOAD` + `GITEA_TOKEN` 两个 secrets |
|
||||||
|
|
||||||
### 纯 Python wheel
|
### 纯 Python wheel
|
||||||
|
|
||||||
@ -62,7 +203,7 @@ python3 wheel_pipeline.py mypackage.whl -o output/
|
|||||||
# → output/mypackage-1.0.0-py3-none-harmonyos_arm64.whl
|
# → output/mypackage-1.0.0-py3-none-harmonyos_arm64.whl
|
||||||
```
|
```
|
||||||
|
|
||||||
### Native wheel
|
### Native wheel — 交叉编译
|
||||||
|
|
||||||
含 C/C++/Rust 等 native 扩展的 wheel 需要：
|
含 C/C++/Rust 等 native 扩展的 wheel 需要：
|
||||||
|
|
||||||
@ -71,30 +212,43 @@ python3 wheel_pipeline.py mypackage.whl -o output/
|
|||||||
3. Python 头文件与目标 Python 版本匹配
|
3. Python 头文件与目标 Python 版本匹配
|
||||||
4. 所有 C 依赖也已移植到 HarmonyOS
|
4. 所有 C 依赖也已移植到 HarmonyOS
|
||||||
|
|
||||||
当前版本仅输出 stub，待后续完善。
|
**当前已支持：** numpy 1.26.4 全流程交叉编译通过
|
||||||
|
|
||||||
## 通过 Gitea Actions 触发流水线
|
| 阶段 | 状态 |
|
||||||
|
|------|------|
|
||||||
|
| meson setup + compile | ✅ 311 个编译目标全部通过 |
|
||||||
|
| .so → AArch64 ELF | ✅ 19 个 .so，逐个校验 e_machine=0xB7 |
|
||||||
|
| SOABI 修正 | ✅ `cpython-312-x86_64-linux-gnu` → `cpython-312-arm64-linux-ohos` |
|
||||||
|
| wheel 组装 | ✅ 含 19 个 .so + 纯 Python 文件 + metadata |
|
||||||
|
| 体积 | ✅ ~12.9 MB（已排除测试文件 / 构建源码） |
|
||||||
|
|
||||||
本仓库配置了 Gitea Actions 工作流，commit 推送到 `main` 分支后会自动触发。
|
**使用方式：**
|
||||||
|
|
||||||
### 手动触发
|
```bash
|
||||||
|
python3 wheel_pipeline.py --build-from-source numpy --package-version 1.26.4
|
||||||
|
```
|
||||||
|
|
||||||
1. 打开 Gitea 仓库页面 → **Actions** 标签
|
**待收口到正式发布的问题：**
|
||||||
2. 在 **EcoArk Wheel Pipeline CI** 工作流右侧点击 **▸ Run workflow**
|
- SOABI 标签需与目标运行时完全对齐
|
||||||
3. 选择分支为 `main`，点击 **Run workflow**
|
- pyconfig.h 需为 aarch64-linux-ohos + musl 重新生成
|
||||||
|
- 如需 BLAS 性能需交叉编译 OpenBLAS（当前 `-Dallow-noblas=true`）
|
||||||
|
|
||||||
### 工作流说明
|
### 编译环境需求
|
||||||
|
|
||||||
| Job | 作用 |
|
执行 `--build-from-source` 交叉编译需要以下环境（当前构建机器已具备）：
|
||||||
|-----|------|
|
|
||||||
| `validate` | 生成测试 wheel，分别验证纯 Python 轮子的重打包和 native 轮子的重编译入口 |
|
|
||||||
|
|
||||||
工作流文件位于 `.gitea/workflows/wheel-pipeline.yaml`，可直接扩展为生产流水线。
|
| 依赖 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| OHOS NDK clang（aarch64-linux-ohos） | `/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm` |
|
||||||
|
| Cython ≥0.29.34 | pip install |
|
||||||
|
| meson-python ≥0.15.0 | pip install |
|
||||||
|
| ninja | pip install |
|
||||||
|
| setuptools | pip install（Python 3.12+ 需要） |
|
||||||
|
| meson cross-compilation file | `experiments/ohos-aarch64-cross.ini` |
|
||||||
|
|
||||||
## 依赖
|
> 以上依赖在构建机上已安装。如需在 CI 中运行，需在 workflow 中先执行 `pip install` 安装构建时依赖。
|
||||||
|
|
||||||
- Python ≥ 3.8（标准库，无需额外安装包）
|
详细实验记录见 `experiments/` 目录。
|
||||||
- 对 native 包：HarmonyOS 工具链（外部，不在本仓库内）
|
|
||||||
|
|
||||||
## 许可证
|
## 许可证
|
||||||
|
|
||||||
|
|||||||
82
experiments/README.md
Normal file
82
experiments/README.md
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
# NumPy 真实重编实验记录
|
||||||
|
|
||||||
|
## 实验目标
|
||||||
|
|
||||||
|
验证在 x86_64 主机上，使用 OHOS NDK 交叉编译器，为 `aarch64-linux-ohos` (HarmonyOS arm64-v8a) 目标编译 NumPy 1.26.4 完整 native 扩展的真实阻塞点。
|
||||||
|
|
||||||
|
## 实验环境
|
||||||
|
|
||||||
|
- 主机: x86_64 Debian forky/sid, Python 3.12.13
|
||||||
|
- OHOS NDK: `/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm`
|
||||||
|
- clang 15.0.4, target aarch64-linux-ohos
|
||||||
|
- sysroot 包含 musl libc + crt 启动文件
|
||||||
|
- 链接器: ld.lld 15.0.4
|
||||||
|
- Python 3.12 交叉编译产物 (来自 EcoArk): `/home/program/EcoArk/sdk/pythonrunner/src/main/cpp/third_party/python/`
|
||||||
|
|
||||||
|
## 实验方法
|
||||||
|
|
||||||
|
1. pip 创建 venv，安装 Cython 0.29.36 + meson-python 0.15.0 + ninja + setuptools
|
||||||
|
2. 下载 numpy 1.26.4 sdist
|
||||||
|
3. 使用 numpy 自带的 vendored-meson（含自定义 features 模块）
|
||||||
|
4. 编写 OHOS aarch64 meson cross-compilation file
|
||||||
|
5. 执行 `meson setup` → `meson compile`
|
||||||
|
|
||||||
|
## 结果总结
|
||||||
|
|
||||||
|
### 流程进度
|
||||||
|
|
||||||
|
| 步骤 | 状态 | 备注 |
|
||||||
|
|------|------|------|
|
||||||
|
| NDK 交叉编译器可用性 | ✅ | `clang --target=aarch64-linux-ohos` 可编译 C 程序 |
|
||||||
|
| meson setup | ✅ | 需 `longdouble_format=IEEE_QUAD_LE` 绕过目标不可执行的问题 |
|
||||||
|
| meson compile | ✅ | 195 个 target 全部通过，产出 19 个 .so |
|
||||||
|
| 产物的正确性 | ✅ | 全部 19 个 .so 均为 ARM aarch64 ELF |
|
||||||
|
| so 动态链接 | ✅ | 仅依赖 libc.so (musl) |
|
||||||
|
|
||||||
|
### 第一个真实阻塞点
|
||||||
|
|
||||||
|
**`numpy/core/meson.build:376` — 无法在交叉编译环境中运行测试程序**
|
||||||
|
|
||||||
|
- 原因: numpy 编译时需要运行 C 程序检测 `long double` 的内存布局格式（IEEE_DOUBLE_LE / INTEL_EXTENDED_16_BYTES_LE / IEEE_QUAD_LE / IBM_DOUBLE_DOUBLE_LE 等）
|
||||||
|
- 交叉编译时跨架构二进制无法在 build machine 运行
|
||||||
|
- 解决: 在 cross file 的 `[properties]` 中预声明 `longdouble_format = 'IEEE_QUAD_LE'`
|
||||||
|
|
||||||
|
### 次要阻塞点
|
||||||
|
|
||||||
|
| 阻塞点 | 位置 | 解决方式 |
|
||||||
|
|--------|------|----------|
|
||||||
|
| Cython 0.29 依赖 distutils (Python 3.12 移除) | Cython → distutils.extension | `pip install setuptools` |
|
||||||
|
| meson 缺少 "features" 模块 | meson_cpu/x86/meson.build:2 | 必须使用 numpy 自带的 vendored-meson |
|
||||||
|
| 交叉编译时 sizeof 类检测跳过 | 多个 meson.build 位置 | 系统 meson 自动处理 |
|
||||||
|
| OpenBLAS 未找到 | numpy/core/meson.build | `-Dallow-noblas=true` 降级 |
|
||||||
|
|
||||||
|
### 待解决（未阻塞编译，但影响最终产物）
|
||||||
|
|
||||||
|
1. **SOABI 标签修正**: .so 文件名当前为 `cpython-312-x86_64-linux-gnu`(来自 build machine Python)，需改为 `cpython-312-aarch64-linux-ohos`（对应目标平台的 SOABI）
|
||||||
|
2. **Wheel 打包**: 需将纯 Python 文件（来自 source tree）与交叉编译 .so 组合为 wheel，修改 tag 为 `harmonyos_arm64`
|
||||||
|
3. **OpenBLAS**: numpy 当前使用内置 slow fallback BLAS（lapack_lite），如需性能需交叉编译 OpenBLAS
|
||||||
|
4. **pyconfig.h 校准**: 当前使用 x86_64 原生 configure 生成的 pyconfig.h，需针对 aarch64-linux-ohos + musl 重新生成
|
||||||
|
|
||||||
|
## 最小前置依赖清单
|
||||||
|
|
||||||
|
```
|
||||||
|
pip install Cython>=0.29.34,<3.1 meson-python>=0.15.0,<0.16.0 ninja setuptools
|
||||||
|
# 使用 numpy vendored meson（内置 features 模块）
|
||||||
|
# cross file: ohos-aarch64-cross.ini
|
||||||
|
# meson args: -Dallow-noblas=true -Ddisable-threading=true -Ddisable-optimization=true
|
||||||
|
```
|
||||||
|
|
||||||
|
## 参考命令
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 完整编译流程（已验证通过）
|
||||||
|
pip install Cython==0.29.36 meson-python==0.15.0 ninja setuptools
|
||||||
|
tar xzf numpy-1.26.4.tar.gz
|
||||||
|
cd numpy-1.26.4
|
||||||
|
# 必须用 vendored meson！
|
||||||
|
python3 vendored-meson/meson/meson.py setup build \
|
||||||
|
--cross-file ../ohos-aarch64-cross.ini \
|
||||||
|
-Dallow-noblas=true -Ddisable-threading=true -Ddisable-optimization=true
|
||||||
|
python3 vendored-meson/meson/meson.py compile -C build -j4
|
||||||
|
# 产出在 build/numpy/ 下
|
||||||
|
```
|
||||||
43
experiments/ohos-aarch64-cross.ini
Normal file
43
experiments/ohos-aarch64-cross.ini
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
# Meson cross-compilation file for OHOS aarch64 (arm64-v8a)
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# cd numpy-1.26.4
|
||||||
|
# meson setup build \
|
||||||
|
# --cross-file /path/to/ohos-aarch64-cross.ini \
|
||||||
|
# -Dallow-noblas=true \
|
||||||
|
# -Ddisable-threading=true \
|
||||||
|
# -Ddisable-optimization=true
|
||||||
|
#
|
||||||
|
# NOTE: numpy 1.26.x requires VENDORED meson (not system meson).
|
||||||
|
# Use: python3 vendored-meson/meson/meson.py setup ...
|
||||||
|
#
|
||||||
|
# Toolchain path (adjust as needed):
|
||||||
|
# /home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm
|
||||||
|
#
|
||||||
|
# Sysroot:
|
||||||
|
# /home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot
|
||||||
|
|
||||||
|
[binaries]
|
||||||
|
c = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/clang'
|
||||||
|
cpp = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/clang++'
|
||||||
|
ar = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/llvm-ar'
|
||||||
|
strip = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/llvm-strip'
|
||||||
|
ld = '/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm/bin/ld.lld'
|
||||||
|
|
||||||
|
[built-in options]
|
||||||
|
c_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot', '-D__MUSL__']
|
||||||
|
c_link_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot']
|
||||||
|
cpp_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot', '-D__MUSL__']
|
||||||
|
cpp_link_args = ['--target=aarch64-linux-ohos', '--sysroot=/home/program/tools/command-line-tools/sdk/default/openharmony/native/sysroot']
|
||||||
|
|
||||||
|
[properties]
|
||||||
|
skip_sanity_check = true
|
||||||
|
allow-noblas = true
|
||||||
|
# aarch64-linux-ohos uses MUSL which defines IEEE quad (128-bit) long double
|
||||||
|
longdouble_format = 'IEEE_QUAD_LE'
|
||||||
|
|
||||||
|
[host_machine]
|
||||||
|
system = 'linux'
|
||||||
|
cpu_family = 'aarch64'
|
||||||
|
cpu = 'armv8-a'
|
||||||
|
endian = 'little'
|
||||||
18
packages.txt
Normal file
18
packages.txt
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
# packages.txt — EcoArk HarmonyOS wheel rebuild manifest
|
||||||
|
#
|
||||||
|
# ── Format ──────────────────────────────────────────────
|
||||||
|
# package_name:git_repository_url
|
||||||
|
#
|
||||||
|
# Each uncommented line defines one package to rebuild.
|
||||||
|
# The CI pipeline clones the source, runs the HarmonyOS/arm64
|
||||||
|
# cross-compilation flow, and bundles all output .whl files
|
||||||
|
# into a single archive.
|
||||||
|
#
|
||||||
|
# ── Rules ───────────────────────────────────────────────
|
||||||
|
# * Empty lines and lines starting with # are ignored.
|
||||||
|
# * package_name — short identifier used for output filenames
|
||||||
|
# * repo_url — any `git clone`-compatible URL
|
||||||
|
#
|
||||||
|
# ── Examples (commented out — uncomment to activate) ───
|
||||||
|
numpy:https://github.com/numpy/numpy.git
|
||||||
|
# scipy:https://github.com/scipy/scipy.git
|
||||||
121
pipeline/cli.py
121
pipeline/cli.py
@ -1,22 +1,131 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
import glob
|
||||||
import os
|
import os
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
from . import detect, repack, rebuild
|
from . import detect, repack, rebuild
|
||||||
|
from .manifest import parse_manifest
|
||||||
|
|
||||||
|
# Lazy import for integration staging (optional dependency)
|
||||||
|
try:
|
||||||
|
from scripts.stage_integration import stage_integration as _stage_integration
|
||||||
|
except ImportError:
|
||||||
|
_stage_integration = None
|
||||||
|
|
||||||
|
BUILD_RECIPES = {
|
||||||
|
"numpy": rebuild.build_numpy_wheel,
|
||||||
|
}
|
||||||
|
|
||||||
|
MANIFEST_SOURCES_DIR = "output/sources"
|
||||||
|
|
||||||
|
|
||||||
|
def _clone_repo(repo_url, pkg_name):
|
||||||
|
dest = os.path.join(MANIFEST_SOURCES_DIR, pkg_name)
|
||||||
|
if os.path.isdir(dest) and os.listdir(dest):
|
||||||
|
print(f"[manifest] Source already exists: {dest}")
|
||||||
|
return dest
|
||||||
|
os.makedirs(dest, exist_ok=True)
|
||||||
|
try:
|
||||||
|
print(f"[manifest] Cloning {repo_url} \u2192 {dest}")
|
||||||
|
subprocess.run(
|
||||||
|
["git", "clone", "--depth", "1", repo_url, dest],
|
||||||
|
check=True, capture_output=True, timeout=120,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[manifest] Warning: git clone failed for {pkg_name}: {e}")
|
||||||
|
return dest
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_wheels(wheel_dir, output_dir, zip_name="ecoark-rebuild-output.zip"):
|
||||||
|
whls = sorted(glob.glob(os.path.join(wheel_dir, "*.whl")))
|
||||||
|
zip_path = os.path.join(output_dir, zip_name)
|
||||||
|
os.makedirs(os.path.dirname(zip_path), exist_ok=True)
|
||||||
|
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||||
|
for whl in whls:
|
||||||
|
zf.write(whl, os.path.basename(whl))
|
||||||
|
print(f"[manifest] Bundle: {zip_path} ({len(whls)} wheel(s))")
|
||||||
|
return zip_path
|
||||||
|
|
||||||
|
|
||||||
|
def run_manifest(manifest_path, output_dir, target_os="harmonyos", arch="arm64"):
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
entries = parse_manifest(manifest_path)
|
||||||
|
if not entries:
|
||||||
|
print(f"[manifest] No active entries in {manifest_path} \u2014 nothing to build.")
|
||||||
|
print("[manifest] Edit packages.txt or create a temporary manifest to proceed.")
|
||||||
|
_collect_wheels(output_dir, output_dir)
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"[manifest] Processing {len(entries)} package(s) from {manifest_path}")
|
||||||
|
wheel_dir = os.path.join(output_dir, "wheels")
|
||||||
|
|
||||||
|
for pkg_name, repo_url in entries:
|
||||||
|
print(f"\n{'=' * 60}")
|
||||||
|
print(f"[manifest] Package: {pkg_name}")
|
||||||
|
print(f"[manifest] Repo : {repo_url}")
|
||||||
|
src = _clone_repo(repo_url, pkg_name)
|
||||||
|
rebuild.rebuild_from_source(src, pkg_name, wheel_dir, target_os=target_os, arch=arch)
|
||||||
|
|
||||||
|
bundle = _collect_wheels(wheel_dir, output_dir)
|
||||||
|
print(f"\n{'=' * 60}")
|
||||||
|
print(f"[manifest] Done. Bundle: {bundle}")
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="EcoArk HarmonyOS Wheel Pipeline — repack / rebuild Python wheels for HarmonyOS/arm64"
|
description="EcoArk HarmonyOS Wheel Pipeline \u2014 repack / rebuild Python wheels for HarmonyOS/arm64"
|
||||||
)
|
)
|
||||||
parser.add_argument("input", help="Path to a .whl file")
|
parser.add_argument("input", nargs="?", help="Path to a .whl file (single-wheel mode)")
|
||||||
|
parser.add_argument("--manifest", help="Path to packages manifest file (manifest-driven mode)")
|
||||||
|
parser.add_argument("--build-from-source", choices=list(BUILD_RECIPES.keys()),
|
||||||
|
help="Build a package from source (cross-compile for HarmonyOS)")
|
||||||
|
parser.add_argument("--package-version", default="1.26.4",
|
||||||
|
help="Package version for --build-from-source (default: 1.26.4)")
|
||||||
parser.add_argument("--output-dir", "-o", default="output", help="Output directory (default: ./output)")
|
parser.add_argument("--output-dir", "-o", default="output", help="Output directory (default: ./output)")
|
||||||
parser.add_argument("--target-os", default="harmonyos", choices=["harmonyos", "linux", "ohos"])
|
parser.add_argument("--target-os", default="harmonyos", choices=["harmonyos", "linux", "ohos"])
|
||||||
parser.add_argument("--arch", default="arm64", choices=["arm64", "aarch64", "x86_64"])
|
parser.add_argument("--arch", default="arm64", choices=["arm64", "aarch64", "x86_64"])
|
||||||
parser.add_argument("--force-native", action="store_true", help="Treat as native wheel even if detection says pure")
|
parser.add_argument("--force-native", action="store_true", help="Treat as native wheel even if detection says pure")
|
||||||
parser.add_argument("--force-repack", action="store_true", help="Force repack even for native wheels")
|
parser.add_argument("--force-repack", action="store_true", help="Force repack even for native wheels")
|
||||||
|
parser.add_argument("--rebuild", action="store_true", help="Force rebuild even if build dir exists")
|
||||||
|
parser.add_argument("--stage-integration", nargs="?",
|
||||||
|
const="output/wheels/numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl",
|
||||||
|
help="Stage native .so files for EcoArk integration (optional: specify wheel path)")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.build_from_source:
|
||||||
|
builder = BUILD_RECIPES[args.build_from_source]
|
||||||
|
output_dir = os.path.abspath(args.output_dir)
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
wheel_path = builder(
|
||||||
|
version=args.package_version,
|
||||||
|
output_dir=output_dir,
|
||||||
|
rebuild=args.rebuild,
|
||||||
|
)
|
||||||
|
if wheel_path:
|
||||||
|
print(f"[pipeline] Done. Wheel: {wheel_path}")
|
||||||
|
else:
|
||||||
|
print("[pipeline] FAILED", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.manifest:
|
||||||
|
run_manifest(args.manifest, args.output_dir, args.target_os, args.arch)
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.stage_integration:
|
||||||
|
if _stage_integration is None:
|
||||||
|
print("[pipeline] ERROR: scripts.stage_integration not importable", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
wheel = os.path.abspath(args.stage_integration)
|
||||||
|
out = os.path.join(os.path.abspath(args.output_dir), "integration")
|
||||||
|
rc = _stage_integration(wheel, out)
|
||||||
|
sys.exit(rc)
|
||||||
|
|
||||||
|
if not args.input:
|
||||||
|
parser.error("either INPUT (a .whl file), --manifest, --build-from-source, or --stage-integration is required")
|
||||||
|
|
||||||
input_path = args.input
|
input_path = args.input
|
||||||
if not os.path.isfile(input_path):
|
if not os.path.isfile(input_path):
|
||||||
print(f"Error: input file not found: {input_path}", file=sys.stderr)
|
print(f"Error: input file not found: {input_path}", file=sys.stderr)
|
||||||
@ -40,23 +149,23 @@ def main():
|
|||||||
is_native = info["native"] or args.force_native
|
is_native = info["native"] or args.force_native
|
||||||
|
|
||||||
if not is_native:
|
if not is_native:
|
||||||
print("[pipeline] Pure Python wheel — repack with harmonyos tag")
|
print("[pipeline] Pure Python wheel \u2014 repack with harmonyos tag")
|
||||||
out = repack.repack_wheel(input_path, output_dir)
|
out = repack.repack_wheel(input_path, output_dir)
|
||||||
print(f"[pipeline] Output : {out}")
|
print(f"[pipeline] Output : {out}")
|
||||||
print("[pipeline] Done — wheel can be installed directly on HarmonyOS.")
|
print("[pipeline] Done \u2014 wheel can be installed directly on HarmonyOS.")
|
||||||
else:
|
else:
|
||||||
if args.force_repack:
|
if args.force_repack:
|
||||||
print("[pipeline] Pure wheel detected but force-repack requested.")
|
print("[pipeline] Pure wheel detected but force-repack requested.")
|
||||||
out = repack.repack_wheel(input_path, output_dir)
|
out = repack.repack_wheel(input_path, output_dir)
|
||||||
print(f"[pipeline] Repacked : {out}")
|
print(f"[pipeline] Repacked : {out}")
|
||||||
else:
|
else:
|
||||||
print("[pipeline] Native wheel — needs cross-compilation.")
|
print("[pipeline] Native wheel \u2014 needs cross-compilation.")
|
||||||
out = rebuild.rebuild_native_wheel(
|
out = rebuild.rebuild_native_wheel(
|
||||||
input_path, output_dir,
|
input_path, output_dir,
|
||||||
target_os=args.target_os, arch=args.arch,
|
target_os=args.target_os, arch=args.arch,
|
||||||
)
|
)
|
||||||
print(f"[pipeline] Output : {out}")
|
print(f"[pipeline] Output : {out}")
|
||||||
print("[pipeline] Done — native rebuild entry point triggered.")
|
print("[pipeline] Done \u2014 native rebuild entry point triggered.")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
16
pipeline/manifest.py
Normal file
16
pipeline/manifest.py
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
def parse_manifest(path):
|
||||||
|
entries = []
|
||||||
|
with open(path, "r") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
if ":" not in line:
|
||||||
|
print(f"[manifest] Skipping malformed line: {line}")
|
||||||
|
continue
|
||||||
|
name, url = line.split(":", 1)
|
||||||
|
name = name.strip()
|
||||||
|
url = url.strip()
|
||||||
|
if name and url:
|
||||||
|
entries.append((name, url))
|
||||||
|
return entries
|
||||||
@ -1,46 +1,408 @@
|
|||||||
|
"""Real cross-compilation + wheel assembly for HarmonyOS/arm64."""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
# ── Paths ─────────────────────────────────────────────────
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
REPO_ROOT = os.path.dirname(HERE)
|
||||||
|
CROSS_FILE = os.path.join(REPO_ROOT, "experiments", "ohos-aarch64-cross.ini")
|
||||||
|
OHOS_NDK = "/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm"
|
||||||
|
SOURCES_DIR = "/tmp/ecoark-sources"
|
||||||
|
OUTPUT_DIR = os.path.join(REPO_ROOT, "output", "wheels")
|
||||||
|
|
||||||
|
|
||||||
def rebuild_native_wheel(wheel_path: str, output_dir: str, target_os: str = "harmonyos", arch: str = "arm64") -> str:
|
def _check_toolchain():
|
||||||
basename = os.path.basename(wheel_path)
|
if not os.path.isdir(OHOS_NDK):
|
||||||
print(f"[rebuild] Native wheel rebuild triggered: {basename}")
|
print(f"[rebuild] ERROR: OHOS NDK not found at {OHOS_NDK}")
|
||||||
print(f"[rebuild] Target: {target_os}/{arch}")
|
print("[rebuild] Set OHOS_NDK_PATH or install the NDK first.")
|
||||||
print(f"[rebuild] Output: {output_dir}")
|
return False
|
||||||
print()
|
for exe in ["clang", "clang++", "ld.lld"]:
|
||||||
print(" Native extension rebuild for HarmonyOS requires:")
|
path = os.path.join(OHOS_NDK, "bin", exe)
|
||||||
print(" - HarmonyOS SDK / OHOS NDK")
|
if not os.path.isfile(path):
|
||||||
print(" - Cross-compilation toolchain for arm64")
|
print(f"[rebuild] Missing tool: {path}")
|
||||||
print(" - Python headers matching the target Python build")
|
return False
|
||||||
print(" - All C/Fortran source dependencies ported to HarmonyOS")
|
return True
|
||||||
print()
|
|
||||||
print(" This entry point will be fleshed out in a future iteration.")
|
|
||||||
print(" For now, the source wheel is copied as-is for manual inspection.")
|
|
||||||
print()
|
|
||||||
|
|
||||||
output_path = os.path.join(output_dir, basename)
|
|
||||||
|
def _ensure_dirs(path):
|
||||||
|
os.makedirs(path, exist_ok=True)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _download_sdist(pkg_name, version, dest_dir):
|
||||||
|
"""Download a PyPI source tarball (sdist) for the given package/version."""
|
||||||
|
os.makedirs(dest_dir, exist_ok=True)
|
||||||
|
url = f"https://files.pythonhosted.org/packages/source/{pkg_name[0]}/{pkg_name}/{pkg_name}-{version}.tar.gz"
|
||||||
|
dest = os.path.join(dest_dir, f"{pkg_name}-{version}.tar.gz")
|
||||||
|
if os.path.isfile(dest):
|
||||||
|
print(f"[rebuild] Source tarball already cached: {dest}")
|
||||||
|
return dest
|
||||||
|
print(f"[rebuild] Downloading {url} ...")
|
||||||
|
ret = subprocess.run(
|
||||||
|
["curl", "-sL", url, "--connect-timeout", "10", "--max-time", "300", "-o", dest],
|
||||||
|
capture_output=True, timeout=310,
|
||||||
|
)
|
||||||
|
if ret.returncode != 0:
|
||||||
|
raise RuntimeError(f"Download failed: {ret.stderr.decode()}")
|
||||||
|
print(f"[rebuild] Downloaded: {dest} ({os.path.getsize(dest)} bytes)")
|
||||||
|
return dest
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_tarball(tarball_path, dest_dir):
|
||||||
|
"""Extract a .tar.gz to dest_dir and return the top-level dir name."""
|
||||||
|
print(f"[rebuild] Extracting {tarball_path} ...")
|
||||||
|
with tarfile.open(tarball_path, "r:gz") as tf:
|
||||||
|
top = tf.getnames()[0].split("/")[0]
|
||||||
|
tf.extractall(path=dest_dir)
|
||||||
|
extracted = os.path.join(dest_dir, top)
|
||||||
|
print(f"[rebuild] Extracted to: {extracted}")
|
||||||
|
return extracted
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_dir_empty(path):
|
||||||
|
if os.path.isdir(path):
|
||||||
|
shutil.rmtree(path)
|
||||||
|
os.makedirs(path, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _run(cmd, cwd=None, desc=None):
|
||||||
|
if desc:
|
||||||
|
print(f"[rebuild] {desc} ...")
|
||||||
|
print(f"[rebuild] Running: {' '.join(cmd[:4])} ...")
|
||||||
|
ret = subprocess.run(cmd, cwd=cwd, capture_output=False, timeout=1800)
|
||||||
|
if ret.returncode != 0:
|
||||||
|
print(f"[rebuild] FAILED (exit={ret.returncode})")
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def cross_compile_numpy(source_dir, build_dir, cross_file=None, jobs=None):
|
||||||
|
"""Cross-compile numpy extensions using vendored meson + OHOS cross file.
|
||||||
|
|
||||||
|
Returns (success, build_dir).
|
||||||
|
"""
|
||||||
|
cf = cross_file or CROSS_FILE
|
||||||
|
if not os.path.isfile(cf):
|
||||||
|
print(f"[rebuild] Cross file not found: {cf}")
|
||||||
|
return False, build_dir
|
||||||
|
|
||||||
|
_ensure_dir_empty(build_dir)
|
||||||
|
|
||||||
|
vendored_meson = os.path.join(source_dir, "vendored-meson", "meson", "meson.py")
|
||||||
|
if not os.path.isfile(vendored_meson):
|
||||||
|
# try system meson as fallback
|
||||||
|
meson_cmd = [sys.executable, "-m", "mesonbuild"]
|
||||||
|
if not shutil.which("meson"):
|
||||||
|
print(f"[rebuild] vendored meson not found at {vendored_meson}, and no system meson")
|
||||||
|
return False, build_dir
|
||||||
|
meson_cmd = ["meson"]
|
||||||
|
else:
|
||||||
|
meson_cmd = [sys.executable, vendored_meson]
|
||||||
|
|
||||||
|
njobs = jobs or os.cpu_count() or 4
|
||||||
|
|
||||||
|
# meson setup
|
||||||
|
setup_args = meson_cmd + [
|
||||||
|
"setup", build_dir,
|
||||||
|
f"--cross-file={cf}",
|
||||||
|
"-Dallow-noblas=true",
|
||||||
|
"-Ddisable-threading=true",
|
||||||
|
"-Ddisable-optimization=true",
|
||||||
|
]
|
||||||
|
if not _run(setup_args, cwd=source_dir, desc="meson setup"):
|
||||||
|
return False, build_dir
|
||||||
|
|
||||||
|
# meson compile
|
||||||
|
compile_args = meson_cmd + ["compile", "-C", build_dir, f"-j{njobs}"]
|
||||||
|
if not _run(compile_args, desc="meson compile"):
|
||||||
|
return False, build_dir
|
||||||
|
|
||||||
|
print(f"[rebuild] Cross-compilation succeeded. Build dir: {build_dir}")
|
||||||
|
return True, build_dir
|
||||||
|
|
||||||
|
|
||||||
|
def _should_include_source(fn, rel_path):
|
||||||
|
"""Decide whether a file from the source tree should go into the wheel."""
|
||||||
|
ext = os.path.splitext(fn)[1]
|
||||||
|
|
||||||
|
parts = rel_path.split("/")
|
||||||
|
|
||||||
|
# Exclude tests and benchmarks
|
||||||
|
if "tests" in parts or "benchmarks" in parts:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Exclude C/Cython source directories
|
||||||
|
if "src" in parts and ext in (".py", ".pyx", ".pxd", ".pxi"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Exclude build system files / source code
|
||||||
|
if ext in (".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".pyx", ".pxd", ".pxi"):
|
||||||
|
return False
|
||||||
|
if fn == "meson.build" or fn == "meson_options.txt":
|
||||||
|
return False
|
||||||
|
if fn.startswith("meson-"):
|
||||||
|
return False
|
||||||
|
if fn == "CMakeLists.txt" or fn.endswith(".cmake"):
|
||||||
|
return False
|
||||||
|
if fn.endswith(".in"):
|
||||||
|
return False
|
||||||
|
if fn == "setup.py" or fn == "setup.cfg":
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Exclude __pycache__
|
||||||
|
if "__pycache__" in parts:
|
||||||
|
return False
|
||||||
|
if fn.endswith(".pyc") or fn.endswith(".pyo"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Exclude hidden files
|
||||||
|
if fn.startswith("."):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Exclude top-level config/doc files
|
||||||
|
if len(parts) == 1 and ext in (".cfg", ".ini", ".toml", ".txt", ".rst", ".md",
|
||||||
|
".yml", ".yaml", ".svg", ".png", ".jpg"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _soabi_for(target_os, arch):
|
||||||
|
"""Return the SOABI string for the target platform."""
|
||||||
|
if target_os == "harmonyos" or target_os == "ohos":
|
||||||
|
return f"cpython-312-{arch}-linux-ohos"
|
||||||
|
return f"cpython-312-{arch}-linux-gnu"
|
||||||
|
|
||||||
|
|
||||||
|
def _host_soabi():
|
||||||
|
"""Detect the build machine's SOABI from sysconfig."""
|
||||||
|
import sysconfig
|
||||||
|
return sysconfig.get_config_var("EXT_SUFFIX").lstrip(".").rsplit(".", 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def assemble_wheel(source_dir, build_dir, output_dir, pkg_name, version,
|
||||||
|
target_os="harmonyos", arch="arm64",
|
||||||
|
soabi_override=None):
|
||||||
|
"""Assemble a .whl from pure-Python source + cross-compiled .so files.
|
||||||
|
|
||||||
|
Returns path to the created .whl file.
|
||||||
|
"""
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
with open(wheel_path, "rb") as src, open(output_path, "wb") as dst:
|
target_soabi = soabi_override or _soabi_for(target_os, arch)
|
||||||
while True:
|
host_soabi = _host_soabi()
|
||||||
buf = src.read(65536)
|
|
||||||
if not buf:
|
|
||||||
break
|
|
||||||
dst.write(buf)
|
|
||||||
|
|
||||||
print(f"[rebuild] Copied original wheel to: {output_path}")
|
plat_tag = f"{target_os}_{arch}"
|
||||||
|
wheel_name = f"{pkg_name}-{version}-cp312-cp312-{plat_tag}.whl"
|
||||||
|
wheel_path = os.path.join(output_dir, wheel_name)
|
||||||
|
|
||||||
stub_path = output_path.replace(".whl", f"_{target_os}_{arch}_stub.txt")
|
dist_info = f"{pkg_name}-{version}.dist-info"
|
||||||
with open(stub_path, "w") as f:
|
records = []
|
||||||
f.write(f"Native rebuild stub for {basename} ({target_os}/{arch})\n")
|
|
||||||
f.write("Replace this file with the actual rebuilt harmonyos wheel.\n")
|
|
||||||
|
|
||||||
print(f"[rebuild] Stub marker: {stub_path}")
|
def add_entry(zf, arcname, data_or_path):
|
||||||
|
if isinstance(data_or_path, str) and os.path.isfile(data_or_path):
|
||||||
|
with open(data_or_path, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
elif isinstance(data_or_path, str):
|
||||||
|
data = data_or_path.encode()
|
||||||
|
else:
|
||||||
|
data = data_or_path
|
||||||
|
zf.writestr(arcname, data)
|
||||||
|
digest = hashlib.sha256(data).digest()
|
||||||
|
b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
|
||||||
|
records.append(f"{arcname},sha256={b64},{len(data)}")
|
||||||
|
|
||||||
|
def add_symlink(zf, arcname, target):
|
||||||
|
info = zipfile.ZipInfo(arcname)
|
||||||
|
info.external_attr = 0o120777 << 16
|
||||||
|
zf.writestr(info, target)
|
||||||
|
records.append(f"{arcname},,0")
|
||||||
|
|
||||||
|
with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||||
|
# ── dist-info metadata ──────────────────────────
|
||||||
|
add_entry(zf, f"{dist_info}/METADATA", (
|
||||||
|
f"Metadata-Version: 2.1\n"
|
||||||
|
f"Name: {pkg_name}\n"
|
||||||
|
f"Version: {version}\n"
|
||||||
|
f"Summary: {pkg_name} \u2014 EcoArk HarmonyOS experimental cross-build\n"
|
||||||
|
f"Description: EcoArk HarmonyOS experimental cross-build for {target_os}/{arch}\n"
|
||||||
|
f" Built from source: {source_dir}\n"
|
||||||
|
f" Cross-compiled with: OHOS NDK clang 15.0.4\n"
|
||||||
|
f" This is an EXPERIMENTAL build \u2014 not yet formally released.\n"
|
||||||
|
f"License: See package license\n"
|
||||||
|
f"Requires-Python: >=3.9\n"
|
||||||
|
))
|
||||||
|
add_entry(zf, f"{dist_info}/WHEEL", (
|
||||||
|
f"Wheel-Version: 1.0\n"
|
||||||
|
f"Generator: ecoark-pipeline 0.1\n"
|
||||||
|
f"Root-Is-Purelib: false\n"
|
||||||
|
f"Tag: cp312-cp312-{plat_tag}\n"
|
||||||
|
))
|
||||||
|
add_entry(zf, f"{dist_info}/top_level.txt", f"{pkg_name}\n")
|
||||||
|
add_entry(zf, f"{dist_info}/INSTALLER", "ecoark-pipeline\n")
|
||||||
|
add_entry(zf, f"{dist_info}/entry_points.txt",
|
||||||
|
"[console_scripts]\nf2py = numpy.f2py.f2py2e:main\n")
|
||||||
|
|
||||||
|
# ── Copy Python source files ─────────────────────
|
||||||
|
pkg_src = os.path.join(source_dir, pkg_name)
|
||||||
|
if os.path.isdir(pkg_src):
|
||||||
|
for dirpath, dirnames, filenames in os.walk(pkg_src):
|
||||||
|
rel = os.path.relpath(dirpath, source_dir).replace("\\", "/")
|
||||||
|
# Prune __pycache__
|
||||||
|
if "__pycache__" in dirnames:
|
||||||
|
dirnames.remove("__pycache__")
|
||||||
|
for fn in filenames:
|
||||||
|
src = os.path.join(dirpath, fn)
|
||||||
|
arcname = f"{rel}/{fn}"
|
||||||
|
if _should_include_source(fn, arcname):
|
||||||
|
add_entry(zf, arcname, src)
|
||||||
|
|
||||||
|
# Copy top-level .py files too (e.g., _config.py)
|
||||||
|
for fn in os.listdir(source_dir):
|
||||||
|
if fn.endswith(".py"):
|
||||||
|
src = os.path.join(source_dir, fn)
|
||||||
|
if os.path.isfile(src):
|
||||||
|
add_entry(zf, fn, src)
|
||||||
|
|
||||||
|
# ── Copy compiled .so files (rename SOABI) ──────
|
||||||
|
build_pkg = os.path.join(build_dir, pkg_name)
|
||||||
|
so_count = 0
|
||||||
|
if os.path.isdir(build_pkg):
|
||||||
|
for dirpath, dirnames, filenames in os.walk(build_pkg):
|
||||||
|
rel = os.path.relpath(dirpath, build_dir).replace("\\", "/")
|
||||||
|
for fn in filenames:
|
||||||
|
if not fn.endswith(".so"):
|
||||||
|
continue
|
||||||
|
src = os.path.join(dirpath, fn)
|
||||||
|
new_fn = fn.replace(host_soabi, target_soabi)
|
||||||
|
arcname = f"{rel}/{new_fn}"
|
||||||
|
add_entry(zf, arcname, src)
|
||||||
|
so_count += 1
|
||||||
|
|
||||||
|
# ── Also check for .so files in sub-packages directly
|
||||||
|
# in the build dir (some builds put them differently)
|
||||||
|
for dirpath, dirnames, filenames in os.walk(build_dir):
|
||||||
|
rel = os.path.relpath(dirpath, build_dir).replace("\\", "/")
|
||||||
|
for fn in filenames:
|
||||||
|
if not fn.endswith(".so"):
|
||||||
|
continue
|
||||||
|
if not fn.startswith(pkg_name) and rel not in (".", ""):
|
||||||
|
continue
|
||||||
|
src = os.path.join(dirpath, fn)
|
||||||
|
new_fn = fn.replace(host_soabi, target_soabi)
|
||||||
|
arcname = f"{rel}/{new_fn}"
|
||||||
|
# Only add if not already added
|
||||||
|
if arcname not in [r.split(",")[0] for r in records]:
|
||||||
|
add_entry(zf, arcname, src)
|
||||||
|
so_count += 1
|
||||||
|
|
||||||
|
if so_count == 0:
|
||||||
|
print(f"[rebuild] WARNING: no .so files found in build dir {build_dir}")
|
||||||
|
|
||||||
|
# ── RECORD (final) ─────────────────────────────
|
||||||
|
record_content = "\n".join(records) + "\n"
|
||||||
|
add_entry(zf, f"{dist_info}/RECORD", record_content)
|
||||||
|
|
||||||
|
print(f"[rebuild] Wheel: {wheel_path}")
|
||||||
|
print(f"[rebuild] Size : {os.path.getsize(wheel_path)} bytes")
|
||||||
|
print(f"[rebuild] .so : {so_count} files (SOABI: {host_soabi} -> {target_soabi})")
|
||||||
|
return wheel_path
|
||||||
|
|
||||||
|
|
||||||
|
def build_numpy_wheel(version="1.26.4", output_dir=None, rebuild=False):
|
||||||
|
"""Full numpy cross-compilation + wheel assembly pipeline.
|
||||||
|
|
||||||
|
Steps:
|
||||||
|
1. Download + extract numpy source sdist from PyPI
|
||||||
|
2. Cross-compile with vendored meson + OHOS cross file
|
||||||
|
3. Assemble wheel from Python source + cross-compiled .so files
|
||||||
|
"""
|
||||||
|
if not _check_toolchain():
|
||||||
|
return None
|
||||||
|
|
||||||
|
out_dir = output_dir or OUTPUT_DIR
|
||||||
|
sources = _ensure_dirs(SOURCES_DIR)
|
||||||
|
|
||||||
|
# Download + extract
|
||||||
|
tarball = _download_sdist("numpy", version, sources)
|
||||||
|
src_dir = _extract_tarball(tarball, sources)
|
||||||
|
|
||||||
|
build_dir = os.path.join(sources, f"numpy-{version}-build")
|
||||||
|
if os.path.isdir(build_dir) and not rebuild:
|
||||||
|
print(f"[rebuild] Build dir already exists: {build_dir}")
|
||||||
|
print("[rebuild] Set rebuild=True or remove the dir to rebuild from scratch")
|
||||||
|
else:
|
||||||
|
ok, build_dir = cross_compile_numpy(src_dir, build_dir)
|
||||||
|
if not ok:
|
||||||
|
print("[rebuild] Cross-compilation FAILED")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Assemble wheel
|
||||||
|
wheel_path = assemble_wheel(src_dir, build_dir, out_dir,
|
||||||
|
pkg_name="numpy", version=version,
|
||||||
|
target_os="harmonyos", arch="arm64")
|
||||||
|
return wheel_path
|
||||||
|
|
||||||
|
|
||||||
|
def rebuild_from_source(source_dir, pkg_name, output_dir, version="0.0.0",
|
||||||
|
target_os="harmonyos", arch="arm64"):
|
||||||
|
"""Generic entry point: rebuild a source package for HarmonyOS.
|
||||||
|
|
||||||
|
For now, dispatches to package-specific builders.
|
||||||
|
Falls back to stub wheel generation for unknown packages.
|
||||||
|
"""
|
||||||
|
if pkg_name == "numpy":
|
||||||
|
return build_numpy_wheel(version=version, output_dir=output_dir)
|
||||||
|
return _stub_wheel(source_dir, pkg_name, output_dir, version, target_os, arch)
|
||||||
|
|
||||||
|
|
||||||
|
def rebuild_native_wheel(wheel_path: str, output_dir: str,
|
||||||
|
target_os: str = "harmonyos", arch: str = "arm64") -> str:
|
||||||
|
"""Entry point for wheel-to-wheel rebuild (not yet implemented)."""
|
||||||
|
print(f"[rebuild] Native wheel rebuild from wheel not yet supported: {wheel_path}")
|
||||||
|
print("[rebuild] Use --manifest or rebuild_from_source() instead.")
|
||||||
|
basename = os.path.basename(wheel_path)
|
||||||
|
output_path = os.path.join(output_dir, basename)
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
shutil.copy2(wheel_path, output_path)
|
||||||
return output_path
|
return output_path
|
||||||
|
|
||||||
|
|
||||||
|
def _stub_wheel(source_dir, pkg_name, output_dir, version, target_os, arch):
|
||||||
|
"""Generate a stub wheel with metadata only."""
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
wheel_name = f"{pkg_name}-{version}-cp312-cp312-{target_os}_{arch}.whl"
|
||||||
|
wheel_path = os.path.join(output_dir, wheel_name)
|
||||||
|
dist_info = f"{pkg_name}-{version}.dist-info"
|
||||||
|
with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||||
|
zf.writestr(f"{dist_info}/METADATA",
|
||||||
|
f"Metadata-Version: 2.1\nName: {pkg_name}\nVersion: {version}\n"
|
||||||
|
f"Summary: EcoArk HarmonyOS rebuild (stub)\n")
|
||||||
|
zf.writestr(f"{dist_info}/WHEEL",
|
||||||
|
f"Wheel-Version: 1.0\nGenerator: ecoark-pipeline\n"
|
||||||
|
f"Root-Is-Purelib: false\nTag: cp312-cp312-{target_os}_{arch}\n")
|
||||||
|
zf.writestr(f"{dist_info}/RECORD", "")
|
||||||
|
zf.writestr(f"{pkg_name}/__init__.py",
|
||||||
|
f"def hello(): return 'built by EcoArk pipeline (stub)'\n")
|
||||||
|
zf.writestr(f"{pkg_name}/_ecoark_stub.txt",
|
||||||
|
f"Package: {pkg_name}\nSource: {source_dir}\n"
|
||||||
|
f"Target: {target_os}/{arch}\nStub wheel (replace with real build).\n")
|
||||||
|
print(f"[rebuild] Stub wheel: {wheel_path}")
|
||||||
|
return wheel_path
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
if len(sys.argv) < 2:
|
import sys
|
||||||
print("Usage: python -m pipeline.rebuild <wheel.whl> [output_dir]")
|
if len(sys.argv) > 1 and sys.argv[1] == "numpy":
|
||||||
sys.exit(1)
|
build_numpy_wheel(version=sys.argv[2] if len(sys.argv) > 2 else "1.26.4")
|
||||||
rebuild_native_wheel(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "output")
|
else:
|
||||||
|
print("Usage: python -m pipeline.rebuild numpy [version]")
|
||||||
|
print(" python -m pipeline.rebuild numpy 1.26.4")
|
||||||
|
|||||||
31
recipes/numpy_integration_manifest.json
Normal file
31
recipes/numpy_integration_manifest.json
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"pkg": "numpy",
|
||||||
|
"version": "1.26.4",
|
||||||
|
"description": "Integration staging recipe for numpy native .so modules into EcoArk pythonrunner main-so linking",
|
||||||
|
"staging": {
|
||||||
|
"source_wheel": "output/wheels/numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl",
|
||||||
|
"output_dir": "output/integration/numpy"
|
||||||
|
},
|
||||||
|
"modules": [
|
||||||
|
{"so": "numpy/core/_multiarray_umath.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_multiarray_umath.so"},
|
||||||
|
{"so": "numpy/core/_multiarray_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_multiarray_tests.so"},
|
||||||
|
{"so": "numpy/core/_simd.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_simd.so"},
|
||||||
|
{"so": "numpy/core/_operand_flag_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_operand_flag_tests.so"},
|
||||||
|
{"so": "numpy/core/_struct_ufunc_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_struct_ufunc_tests.so"},
|
||||||
|
{"so": "numpy/core/_rational_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_rational_tests.so"},
|
||||||
|
{"so": "numpy/core/_umath_tests.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/core/_umath_tests.so"},
|
||||||
|
{"so": "numpy/fft/_pocketfft_internal.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/fft/_pocketfft_internal.so"},
|
||||||
|
{"so": "numpy/linalg/_umath_linalg.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/linalg/_umath_linalg.so"},
|
||||||
|
{"so": "numpy/linalg/lapack_lite.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/linalg/lapack_lite.so"},
|
||||||
|
{"so": "numpy/random/_generator.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_generator.so"},
|
||||||
|
{"so": "numpy/random/mtrand.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/mtrand.so"},
|
||||||
|
{"so": "numpy/random/_common.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_common.so"},
|
||||||
|
{"so": "numpy/random/_bounded_integers.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_bounded_integers.so"},
|
||||||
|
{"so": "numpy/random/bit_generator.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/bit_generator.so"},
|
||||||
|
{"so": "numpy/random/_sfc64.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_sfc64.so"},
|
||||||
|
{"so": "numpy/random/_pcg64.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_pcg64.so"},
|
||||||
|
{"so": "numpy/random/_philox.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_philox.so"},
|
||||||
|
{"so": "numpy/random/_mt19937.cpython-312-arm64-linux-ohos.so", "dest_rel": "numpy/random/_mt19937.so"}
|
||||||
|
],
|
||||||
|
"ecoark_target_hint": "Copy each .so into EcoArk pythonrunner's native module tree (e.g. lib/python3.12/site-packages/numpy/) so that import numpy works with main-so linking."
|
||||||
|
}
|
||||||
0
scripts/__init__.py
Normal file
0
scripts/__init__.py
Normal file
107
scripts/stage_integration.py
Normal file
107
scripts/stage_integration.py
Normal file
@ -0,0 +1,107 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Stage built native .so files for EcoArk pythonrunner main-so integration.
|
||||||
|
|
||||||
|
Extracts .so files from an existing HarmonyOS-cross-compiled wheel and copies
|
||||||
|
them into an integration output directory with a README describing where each
|
||||||
|
module should be placed in the EcoArk pythonrunner environment.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
|
||||||
|
def stage_integration(wheel_path, output_dir, recipe=None):
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
if not os.path.isfile(wheel_path):
|
||||||
|
print(f"[stage] ERROR: wheel not found: {wheel_path}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
staged = []
|
||||||
|
with zipfile.ZipFile(wheel_path, "r") as z:
|
||||||
|
for name in z.namelist():
|
||||||
|
if not name.endswith(".so"):
|
||||||
|
continue
|
||||||
|
dest = os.path.join(output_dir, name)
|
||||||
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||||
|
with z.open(name) as src, open(dest, "wb") as dst:
|
||||||
|
shutil.copyfileobj(src, dst)
|
||||||
|
staged.append(name)
|
||||||
|
print(f"[stage] {name}")
|
||||||
|
|
||||||
|
readme_path = os.path.join(output_dir, "README.md")
|
||||||
|
with open(readme_path, "w") as f:
|
||||||
|
f.write(f"# Integration Staging for EcoArk pythonrunner main-so linking\n\n")
|
||||||
|
f.write(f"Source wheel: `{wheel_path}`\n\n")
|
||||||
|
f.write(f"## Staged .so files ({len(staged)} total)\n\n")
|
||||||
|
for s in staged:
|
||||||
|
f.write(f"- `{s}`\n")
|
||||||
|
f.write(f"\n## EcoArk target hint\n\n")
|
||||||
|
f.write("Copy each .so into EcoArk pythonrunner's native module tree\n")
|
||||||
|
f.write("under `lib/python3.12/site-packages/` preserving subdirectory\n")
|
||||||
|
f.write("structure so that `import numpy` resolves via main-so linking.\n")
|
||||||
|
so_list_path = os.path.join(output_dir, "integration_manifest.json")
|
||||||
|
manifest = {
|
||||||
|
"pkg": "numpy",
|
||||||
|
"staged_count": len(staged),
|
||||||
|
"ecoark_target": "lib/python3.12/site-packages/",
|
||||||
|
"files": staged,
|
||||||
|
}
|
||||||
|
if recipe and os.path.isfile(recipe):
|
||||||
|
with open(recipe) as rf:
|
||||||
|
manifest["recipe"] = json.load(rf)
|
||||||
|
with open(so_list_path, "w") as f:
|
||||||
|
json.dump(manifest, f, indent=2)
|
||||||
|
|
||||||
|
print(f"\n[stage] Staged {len(staged)} .so files to {output_dir}")
|
||||||
|
print(f"[stage] Manifest: {so_list_path}")
|
||||||
|
print(f"[stage] README: {readme_path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Stage native .so files from a built HarmonyOS wheel for EcoArk integration"
|
||||||
|
)
|
||||||
|
parser.add_argument("wheel", nargs="?", help="Path to built HarmonyOS .whl file")
|
||||||
|
parser.add_argument("--output-dir", "-o", default="output/integration",
|
||||||
|
help="Integration output directory (default: output/integration)")
|
||||||
|
parser.add_argument("--recipe", help="Path to integration recipe JSON (optional)")
|
||||||
|
parser.add_argument("--list-recipes", action="store_true",
|
||||||
|
help="List available recipes and exit")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.list_recipes:
|
||||||
|
recipes_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "recipes")
|
||||||
|
print("Available integration recipes:")
|
||||||
|
if os.path.isdir(recipes_dir):
|
||||||
|
for fn in sorted(os.listdir(recipes_dir)):
|
||||||
|
if fn.endswith(".json"):
|
||||||
|
rp = os.path.join(recipes_dir, fn)
|
||||||
|
with open(rp) as f:
|
||||||
|
d = json.load(f)
|
||||||
|
print(f" {fn} ({d.get('pkg','?')} {d.get('version','?')})")
|
||||||
|
else:
|
||||||
|
print(" (no recipes directory)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if not args.wheel:
|
||||||
|
# Default: look for numpy wheel in output/wheels
|
||||||
|
default = os.path.join(
|
||||||
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
||||||
|
"output", "wheels", "numpy-1.26.4-cp312-cp312-harmonyos_arm64.whl"
|
||||||
|
)
|
||||||
|
if os.path.isfile(default):
|
||||||
|
args.wheel = default
|
||||||
|
print(f"[stage] Using default wheel: {default}")
|
||||||
|
else:
|
||||||
|
parser.error("no wheel specified and default numpy wheel not found")
|
||||||
|
|
||||||
|
return stage_integration(args.wheel, os.path.abspath(args.output_dir), recipe=args.recipe)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
99
scripts/upload_artifacts.py
Normal file
99
scripts/upload_artifacts.py
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Upload build outputs as Gitea Actions artifacts using the Actions Runtime API.
|
||||||
|
|
||||||
|
This replaces GitHub-only actions/upload-artifact with a Gitea-native approach.
|
||||||
|
Uses ACTIONS_RUNTIME_URL and ACTIONS_RUNTIME_TOKEN provided by Gitea's
|
||||||
|
act_runner to upload artifacts directly to the pipeline service.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 scripts/upload_artifacts.py <artifact_name> <path_glob1> [path_glob2 ...]
|
||||||
|
|
||||||
|
Environment (set by Gitea act_runner >= 1.21):
|
||||||
|
ACTIONS_RUNTIME_URL — Pipeline service base URL (e.g. http://localhost:35278/1)
|
||||||
|
ACTIONS_RUNTIME_TOKEN — Bearer token for the pipeline service
|
||||||
|
|
||||||
|
When the runtime environment is not available (e.g. local testing),
|
||||||
|
artifacts are saved to ./artifacts/<name>.zip as a fallback.
|
||||||
|
"""
|
||||||
|
import glob
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
_RUNTIME_UPLOADED = False
|
||||||
|
|
||||||
|
|
||||||
|
def _upload_via_runtime(name, payload, runtime_url, runtime_token):
|
||||||
|
url = "{}/_apis/pipeline/artifacts/{}".format(runtime_url.rstrip("/"), name)
|
||||||
|
headers = {
|
||||||
|
"Authorization": "Bearer " + runtime_token,
|
||||||
|
"Content-Type": "application/octet-stream",
|
||||||
|
"Content-Range": "bytes 0-{}/{}".format(len(payload) - 1, len(payload)),
|
||||||
|
"x-tfs-filelength": str(len(payload)),
|
||||||
|
}
|
||||||
|
req = urllib.request.Request(url, data=payload, headers=headers, method="PUT")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
msg = "[upload-artifact] Uploaded '{}' ({} bytes) to Gitea runtime — HTTP {}"
|
||||||
|
print(msg.format(name, len(payload), resp.status))
|
||||||
|
global _RUNTIME_UPLOADED
|
||||||
|
_RUNTIME_UPLOADED = True
|
||||||
|
return True
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
body = e.read().decode()[:300] if e.fp else ""
|
||||||
|
print("[upload-artifact] Runtime upload failed: HTTP {} — {}".format(e.code, body))
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _save_locally(name, payload):
|
||||||
|
dest = os.path.join("artifacts", "{}.zip".format(name))
|
||||||
|
os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
|
||||||
|
with open(dest, "wb") as f:
|
||||||
|
f.write(payload)
|
||||||
|
print("[upload-artifact] Saved artifact to {} ({} bytes)".format(dest, len(payload)))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) < 3:
|
||||||
|
print("Usage: {} <artifact_name> <path_glob> [path_glob ...]".format(sys.argv[0]))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
artifact_name = sys.argv[1]
|
||||||
|
patterns = sys.argv[2:]
|
||||||
|
|
||||||
|
files = []
|
||||||
|
for pattern in patterns:
|
||||||
|
matched = glob.glob(pattern, recursive=True)
|
||||||
|
files.extend(matched)
|
||||||
|
files = sorted(set(f for f in files if os.path.isfile(f)))
|
||||||
|
|
||||||
|
if not files:
|
||||||
|
print("[upload-artifact] No files found for pattern(s): {}".format(patterns))
|
||||||
|
return
|
||||||
|
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||||
|
for f in files:
|
||||||
|
zf.write(f, os.path.relpath(f))
|
||||||
|
payload = buf.getvalue()
|
||||||
|
|
||||||
|
runtime_url = os.environ.get("ACTIONS_RUNTIME_URL", "").rstrip("/")
|
||||||
|
runtime_token = os.environ.get("ACTIONS_RUNTIME_TOKEN", "")
|
||||||
|
|
||||||
|
if runtime_url and runtime_token:
|
||||||
|
ok = _upload_via_runtime(artifact_name, payload, runtime_url, runtime_token)
|
||||||
|
if not ok:
|
||||||
|
_save_locally(artifact_name, payload)
|
||||||
|
else:
|
||||||
|
print("[upload-artifact] ACTIONS_RUNTIME_URL/TOKEN not available — saving locally.")
|
||||||
|
_save_locally(artifact_name, payload)
|
||||||
|
|
||||||
|
print("[upload-artifact] Done — {} file(s) in '{}'.".format(len(files), artifact_name))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
134
scripts/upload_release_assets.py
Normal file
134
scripts/upload_release_assets.py
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Upload wheel / zip assets to a Gitea Release.
|
||||||
|
|
||||||
|
Environment variables (set by Gitea runner or user):
|
||||||
|
ENABLE_RELEASE_UPLOAD — set to "true" to activate (default: skip)
|
||||||
|
GITEA_TOKEN — Gitea access token with `repo` scope
|
||||||
|
RELEASE_TOKEN — fallback if GITEA_TOKEN is unset
|
||||||
|
GITHUB_SERVER_URL — Gitea server URL (runner-provided)
|
||||||
|
GITHUB_REPOSITORY — owner/repo (runner-provided)
|
||||||
|
GITHUB_REF_NAME — tag name when pushing a tag (runner-provided)
|
||||||
|
GITHUB_SHA — commit SHA (runner-provided)
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 scripts/upload_release_assets.py <assets-dir>
|
||||||
|
"""
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
def env(key, default=""):
|
||||||
|
return os.environ.get(key, default)
|
||||||
|
|
||||||
|
|
||||||
|
def _multipart_formdata(field, filename, filepath):
|
||||||
|
boundary = "----GiteaReleaseBoundary{:x}".format(secrets.randbits(64))
|
||||||
|
with open(filepath, "rb") as f:
|
||||||
|
payload = f.read()
|
||||||
|
body = (
|
||||||
|
"--{}\r\n"
|
||||||
|
'Content-Disposition: form-data; name="{}"; filename="{}"\r\n'
|
||||||
|
"Content-Type: application/octet-stream\r\n\r\n"
|
||||||
|
).format(boundary, field, filename).encode() + payload + (
|
||||||
|
"\r\n--{}--\r\n".format(boundary)
|
||||||
|
).encode()
|
||||||
|
return boundary, body
|
||||||
|
|
||||||
|
|
||||||
|
def _api(method, url, headers, data=None, filepath=None):
|
||||||
|
req_headers = dict(headers)
|
||||||
|
body = None
|
||||||
|
if filepath:
|
||||||
|
boundary, body = _multipart_formdata("attachment", os.path.basename(filepath), filepath)
|
||||||
|
req_headers["Content-Type"] = "multipart/form-data; boundary={}".format(boundary)
|
||||||
|
elif data is not None:
|
||||||
|
body = json.dumps(data).encode()
|
||||||
|
req_headers.setdefault("Content-Type", "application/json")
|
||||||
|
req = urllib.request.Request(url, data=body, headers=req_headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
return resp.status, json.loads(resp.read().decode())
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
snippet = e.read().decode()[:200] if e.fp else ""
|
||||||
|
print(" [release] HTTP {} on {} {}: {}".format(e.code, method, url, snippet))
|
||||||
|
return e.code, None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# ── Guard ──
|
||||||
|
if env("ENABLE_RELEASE_UPLOAD").lower() not in ("true", "1", "yes"):
|
||||||
|
print("[release] ENABLE_RELEASE_UPLOAD not active — skipping.")
|
||||||
|
return
|
||||||
|
|
||||||
|
token = env("GITEA_TOKEN") or env("RELEASE_TOKEN")
|
||||||
|
if not token:
|
||||||
|
print("[release] No token (GITEA_TOKEN / RELEASE_TOKEN) — skipping.")
|
||||||
|
return
|
||||||
|
|
||||||
|
server_url = env("GITHUB_SERVER_URL").rstrip("/")
|
||||||
|
repo = env("GITHUB_REPOSITORY")
|
||||||
|
ref_name = env("GITHUB_REF_NAME")
|
||||||
|
sha = env("GITHUB_SHA")
|
||||||
|
|
||||||
|
if not repo:
|
||||||
|
print("[release] GITHUB_REPOSITORY unset — skipping.")
|
||||||
|
return
|
||||||
|
|
||||||
|
# ── Scan assets ──
|
||||||
|
assets_dir = sys.argv[1] if len(sys.argv) > 1 else "output"
|
||||||
|
whl_files = sorted(
|
||||||
|
glob.glob(os.path.join(assets_dir, "*.whl"))
|
||||||
|
+ glob.glob(os.path.join(assets_dir, "wheels", "*.whl"))
|
||||||
|
)
|
||||||
|
zip_files = sorted(glob.glob(os.path.join(assets_dir, "*.zip")))
|
||||||
|
all_files = whl_files + zip_files
|
||||||
|
|
||||||
|
if not all_files:
|
||||||
|
print("[release] No .whl or .zip found under {}/ — nothing to upload.".format(assets_dir))
|
||||||
|
return
|
||||||
|
|
||||||
|
tag = ref_name or ("build-{}".format(sha[:7]) if sha else "latest")
|
||||||
|
|
||||||
|
# ── Gitea API ──
|
||||||
|
api_base = "{}/api/v1/repos/{}".format(server_url, repo)
|
||||||
|
headers = {"Authorization": "token {}".format(token), "Accept": "application/json"}
|
||||||
|
|
||||||
|
# 1) Get or create release
|
||||||
|
print("[release] Target tag: {}".format(tag))
|
||||||
|
status, release = _api("GET", "{}/releases/tags/{}".format(api_base, tag), headers)
|
||||||
|
if status == 200 and release:
|
||||||
|
release_id = release["id"]
|
||||||
|
print("[release] Found existing release #{} ({})".format(release_id, tag))
|
||||||
|
else:
|
||||||
|
print("[release] Creating release for tag {} ...".format(tag))
|
||||||
|
status, release = _api("POST", "{}/releases".format(api_base), headers, {
|
||||||
|
"tag_name": tag,
|
||||||
|
"target_commitish": sha,
|
||||||
|
"name": "EcoArk Wheels {}".format(tag),
|
||||||
|
"body": "Automated wheel build from commit {}.".format(sha[:7] if sha else "unknown"),
|
||||||
|
"draft": False,
|
||||||
|
"prerelease": True,
|
||||||
|
})
|
||||||
|
if status not in (200, 201):
|
||||||
|
print("[release] Failed to create release — skipping asset upload.")
|
||||||
|
return
|
||||||
|
release_id = release["id"]
|
||||||
|
print("[release] Created release #{} ({})".format(release_id, tag))
|
||||||
|
|
||||||
|
# 2) Upload each asset as an attachment
|
||||||
|
for fpath in all_files:
|
||||||
|
fname = os.path.basename(fpath)
|
||||||
|
print("[release] Uploading {} ...".format(fname))
|
||||||
|
_api("POST", "{}/releases/{}/assets".format(api_base, release_id), headers, filepath=fpath)
|
||||||
|
|
||||||
|
print('[release] Done — {} asset(s) uploaded to "{}".'.format(len(all_files), tag))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user