412 lines
16 KiB
Python
412 lines
16 KiB
Python
"""Real cross-compilation + wheel assembly for HarmonyOS/arm64."""
|
|
|
|
import base64
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import zipfile
|
|
|
|
# ── Paths ─────────────────────────────────────────────────
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
REPO_ROOT = os.path.dirname(HERE)
|
|
CROSS_FILE = os.path.join(REPO_ROOT, "experiments", "ohos-aarch64-cross.ini")
|
|
OHOS_NDK = "/home/program/tools/command-line-tools/sdk/default/openharmony/native/llvm"
|
|
SOURCES_DIR = "/tmp/ecoark-sources"
|
|
OUTPUT_DIR = os.path.join(REPO_ROOT, "output", "wheels")
|
|
|
|
|
|
def _check_toolchain():
|
|
if not os.path.isdir(OHOS_NDK):
|
|
print(f"[rebuild] ERROR: OHOS NDK not found at {OHOS_NDK}")
|
|
print("[rebuild] Set OHOS_NDK_PATH or install the NDK first.")
|
|
return False
|
|
for exe in ["clang", "clang++", "ld.lld"]:
|
|
path = os.path.join(OHOS_NDK, "bin", exe)
|
|
if not os.path.isfile(path):
|
|
print(f"[rebuild] Missing tool: {path}")
|
|
return False
|
|
return True
|
|
|
|
|
|
def _ensure_dirs(path):
|
|
os.makedirs(path, exist_ok=True)
|
|
return path
|
|
|
|
|
|
def _download_sdist(pkg_name, version, dest_dir):
|
|
"""Download a PyPI source tarball (sdist) for the given package/version."""
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
url = f"https://files.pythonhosted.org/packages/source/{pkg_name[0]}/{pkg_name}/{pkg_name}-{version}.tar.gz"
|
|
dest = os.path.join(dest_dir, f"{pkg_name}-{version}.tar.gz")
|
|
if os.path.isfile(dest):
|
|
print(f"[rebuild] Source tarball already cached: {dest}")
|
|
return dest
|
|
print(f"[rebuild] Downloading {url} ...")
|
|
ret = subprocess.run(
|
|
["curl", "-sL", url, "--connect-timeout", "10", "--max-time", "300", "-o", dest],
|
|
capture_output=True, timeout=310,
|
|
)
|
|
if ret.returncode != 0:
|
|
raise RuntimeError(f"Download failed: {ret.stderr.decode()}")
|
|
print(f"[rebuild] Downloaded: {dest} ({os.path.getsize(dest)} bytes)")
|
|
return dest
|
|
|
|
|
|
def _extract_tarball(tarball_path, dest_dir):
|
|
"""Extract a .tar.gz to dest_dir and return the top-level dir name."""
|
|
print(f"[rebuild] Extracting {tarball_path} ...")
|
|
with tarfile.open(tarball_path, "r:gz") as tf:
|
|
top = tf.getnames()[0].split("/")[0]
|
|
tf.extractall(path=dest_dir)
|
|
extracted = os.path.join(dest_dir, top)
|
|
print(f"[rebuild] Extracted to: {extracted}")
|
|
return extracted
|
|
|
|
|
|
def _ensure_dir_empty(path):
|
|
if os.path.isdir(path):
|
|
shutil.rmtree(path)
|
|
os.makedirs(path, exist_ok=True)
|
|
|
|
|
|
def _run(cmd, cwd=None, desc=None):
|
|
if desc:
|
|
print(f"[rebuild] {desc} ...")
|
|
print(f"[rebuild] Running: {' '.join(cmd[:4])} ...")
|
|
ret = subprocess.run(cmd, cwd=cwd, capture_output=False, timeout=1800)
|
|
if ret.returncode != 0:
|
|
print(f"[rebuild] FAILED (exit={ret.returncode})")
|
|
return False
|
|
return True
|
|
|
|
|
|
def cross_compile_numpy(source_dir, build_dir, cross_file=None, jobs=None):
|
|
"""Cross-compile numpy extensions using vendored meson + OHOS cross file.
|
|
|
|
Returns (success, build_dir).
|
|
"""
|
|
cf = cross_file or CROSS_FILE
|
|
if not os.path.isfile(cf):
|
|
print(f"[rebuild] Cross file not found: {cf}")
|
|
return False, build_dir
|
|
|
|
_ensure_dir_empty(build_dir)
|
|
|
|
vendored_meson = os.path.join(source_dir, "vendored-meson", "meson", "meson.py")
|
|
if not os.path.isfile(vendored_meson):
|
|
# try system meson as fallback
|
|
meson_cmd = [sys.executable, "-m", "mesonbuild"]
|
|
if not shutil.which("meson"):
|
|
print(f"[rebuild] vendored meson not found at {vendored_meson}, and no system meson")
|
|
return False, build_dir
|
|
meson_cmd = ["meson"]
|
|
else:
|
|
meson_cmd = [sys.executable, vendored_meson]
|
|
|
|
njobs = jobs or os.cpu_count() or 4
|
|
|
|
# meson setup
|
|
setup_args = meson_cmd + [
|
|
"setup", build_dir,
|
|
f"--cross-file={cf}",
|
|
"-Dallow-noblas=true",
|
|
"-Ddisable-threading=true",
|
|
"-Ddisable-optimization=true",
|
|
]
|
|
if not _run(setup_args, cwd=source_dir, desc="meson setup"):
|
|
return False, build_dir
|
|
|
|
# meson compile
|
|
compile_args = meson_cmd + ["compile", "-C", build_dir, f"-j{njobs}"]
|
|
if not _run(compile_args, desc="meson compile"):
|
|
return False, build_dir
|
|
|
|
print(f"[rebuild] Cross-compilation succeeded. Build dir: {build_dir}")
|
|
return True, build_dir
|
|
|
|
|
|
def _should_include_source(fn, rel_path):
|
|
"""Decide whether a file from the source tree should go into the wheel."""
|
|
ext = os.path.splitext(fn)[1]
|
|
|
|
parts = rel_path.split("/")
|
|
|
|
# Exclude tests and benchmarks
|
|
if "tests" in parts or "benchmarks" in parts:
|
|
return False
|
|
|
|
# Exclude C/Cython source directories
|
|
if "src" in parts and ext in (".py", ".pyx", ".pxd", ".pxi"):
|
|
return False
|
|
|
|
# Exclude build system files / source code
|
|
if ext in (".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".pyx", ".pxd", ".pxi"):
|
|
return False
|
|
if fn == "meson.build" or fn == "meson_options.txt":
|
|
return False
|
|
if fn.startswith("meson-"):
|
|
return False
|
|
if fn == "CMakeLists.txt" or fn.endswith(".cmake"):
|
|
return False
|
|
if fn.endswith(".in"):
|
|
return False
|
|
if fn == "setup.py" or fn == "setup.cfg":
|
|
return False
|
|
|
|
# Exclude __pycache__
|
|
if "__pycache__" in parts:
|
|
return False
|
|
if fn.endswith(".pyc") or fn.endswith(".pyo"):
|
|
return False
|
|
|
|
# Exclude hidden files
|
|
if fn.startswith("."):
|
|
return False
|
|
|
|
# Exclude top-level config/doc files
|
|
if len(parts) == 1 and ext in (".cfg", ".ini", ".toml", ".txt", ".rst", ".md",
|
|
".yml", ".yaml", ".svg", ".png", ".jpg"):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def _soabi_for(target_os, arch):
|
|
"""Return the SOABI string for the target platform."""
|
|
if target_os == "harmonyos" or target_os == "ohos":
|
|
return f"cpython-312-{arch}-linux-ohos"
|
|
return f"cpython-312-{arch}-linux-gnu"
|
|
|
|
|
|
def _host_soabi():
|
|
"""Detect the build machine's SOABI from sysconfig."""
|
|
import sysconfig
|
|
return sysconfig.get_config_var("EXT_SUFFIX").lstrip(".").rsplit(".", 1)[0]
|
|
|
|
|
|
def assemble_wheel(source_dir, build_dir, output_dir, pkg_name, version,
|
|
target_os="harmonyos", arch="arm64",
|
|
soabi_override=None):
|
|
"""Assemble a .whl from pure-Python source + cross-compiled .so files.
|
|
|
|
Returns path to the created .whl file.
|
|
"""
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
target_soabi = soabi_override or _soabi_for(target_os, arch)
|
|
host_soabi = _host_soabi()
|
|
|
|
plat_tag = f"{target_os}_{arch}"
|
|
wheel_name = f"{pkg_name}-{version}-cp312-cp312-{plat_tag}.whl"
|
|
wheel_path = os.path.join(output_dir, wheel_name)
|
|
|
|
dist_info = f"{pkg_name}-{version}.dist-info"
|
|
records = []
|
|
|
|
def add_entry(zf, arcname, data_or_path):
|
|
if isinstance(data_or_path, str) and os.path.isfile(data_or_path):
|
|
with open(data_or_path, "rb") as f:
|
|
data = f.read()
|
|
elif isinstance(data_or_path, str):
|
|
data = data_or_path.encode()
|
|
else:
|
|
data = data_or_path
|
|
zf.writestr(arcname, data)
|
|
digest = hashlib.sha256(data).digest()
|
|
b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
|
|
records.append(f"{arcname},sha256={b64},{len(data)}")
|
|
|
|
def add_symlink(zf, arcname, target):
|
|
info = zipfile.ZipInfo(arcname)
|
|
info.external_attr = 0o120777 << 16
|
|
zf.writestr(info, target)
|
|
records.append(f"{arcname},,0")
|
|
|
|
with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
# ── dist-info metadata ──────────────────────────
|
|
add_entry(zf, f"{dist_info}/METADATA", (
|
|
f"Metadata-Version: 2.1\n"
|
|
f"Name: {pkg_name}\n"
|
|
f"Version: {version}\n"
|
|
f"Summary: {pkg_name} \u2014 EcoArk HarmonyOS experimental cross-build\n"
|
|
f"Description: EcoArk HarmonyOS experimental cross-build for {target_os}/{arch}\n"
|
|
f" Built from source: {source_dir}\n"
|
|
f" Cross-compiled with: OHOS NDK clang 15.0.4\n"
|
|
f" This is an EXPERIMENTAL build \u2014 not yet formally released.\n"
|
|
f"License: See package license\n"
|
|
f"Requires-Python: >=3.9\n"
|
|
))
|
|
add_entry(zf, f"{dist_info}/WHEEL", (
|
|
f"Wheel-Version: 1.0\n"
|
|
f"Generator: ecoark-pipeline 0.1\n"
|
|
f"Root-Is-Purelib: false\n"
|
|
f"Tag: cp312-cp312-{plat_tag}\n"
|
|
))
|
|
add_entry(zf, f"{dist_info}/top_level.txt", f"{pkg_name}\n")
|
|
add_entry(zf, f"{dist_info}/INSTALLER", "ecoark-pipeline\n")
|
|
add_entry(zf, f"{dist_info}/entry_points.txt",
|
|
"[console_scripts]\nf2py = numpy.f2py.f2py2e:main\n")
|
|
|
|
# ── Copy Python source files ─────────────────────
|
|
pkg_src = os.path.join(source_dir, pkg_name)
|
|
if os.path.isdir(pkg_src):
|
|
for dirpath, dirnames, filenames in os.walk(pkg_src):
|
|
rel = os.path.relpath(dirpath, source_dir).replace("\\", "/")
|
|
# Prune __pycache__
|
|
if "__pycache__" in dirnames:
|
|
dirnames.remove("__pycache__")
|
|
for fn in filenames:
|
|
src = os.path.join(dirpath, fn)
|
|
arcname = f"{rel}/{fn}"
|
|
if _should_include_source(fn, arcname):
|
|
add_entry(zf, arcname, src)
|
|
|
|
# Copy top-level .py files too (e.g., _config.py)
|
|
for fn in os.listdir(source_dir):
|
|
if fn.endswith(".py"):
|
|
src = os.path.join(source_dir, fn)
|
|
if os.path.isfile(src):
|
|
add_entry(zf, fn, src)
|
|
|
|
# ── Copy compiled .so files (rename SOABI) ──────
|
|
build_pkg = os.path.join(build_dir, pkg_name)
|
|
so_count = 0
|
|
if os.path.isdir(build_pkg):
|
|
for dirpath, dirnames, filenames in os.walk(build_pkg):
|
|
rel = os.path.relpath(dirpath, build_dir).replace("\\", "/")
|
|
for fn in filenames:
|
|
if not fn.endswith(".so"):
|
|
continue
|
|
src = os.path.join(dirpath, fn)
|
|
new_fn = fn.replace(host_soabi, target_soabi)
|
|
arcname = f"{rel}/{new_fn}"
|
|
add_entry(zf, arcname, src)
|
|
so_count += 1
|
|
|
|
# ── Also check for .so files in sub-packages directly
|
|
# in the build dir (some builds put them differently)
|
|
for dirpath, dirnames, filenames in os.walk(build_dir):
|
|
rel = os.path.relpath(dirpath, build_dir).replace("\\", "/")
|
|
for fn in filenames:
|
|
if not fn.endswith(".so"):
|
|
continue
|
|
if not fn.startswith(pkg_name) and rel not in (".", ""):
|
|
continue
|
|
src = os.path.join(dirpath, fn)
|
|
new_fn = fn.replace(host_soabi, target_soabi)
|
|
arcname = f"{rel}/{new_fn}"
|
|
# Only add if not already added
|
|
if arcname not in [r.split(",")[0] for r in records]:
|
|
add_entry(zf, arcname, src)
|
|
so_count += 1
|
|
|
|
if so_count == 0:
|
|
print(f"[rebuild] WARNING: no .so files found in build dir {build_dir}")
|
|
|
|
# ── RECORD (final) ─────────────────────────────
|
|
record_content = "\n".join(records) + "\n"
|
|
add_entry(zf, f"{dist_info}/RECORD", record_content)
|
|
|
|
print(f"[rebuild] Wheel: {wheel_path}")
|
|
print(f"[rebuild] Size : {os.path.getsize(wheel_path)} bytes")
|
|
print(f"[rebuild] .so : {so_count} files (SOABI: {host_soabi} -> {target_soabi})")
|
|
return wheel_path
|
|
|
|
|
|
def build_numpy_wheel(version="1.26.4", output_dir=None, rebuild=False):
|
|
"""Full numpy cross-compilation + wheel assembly pipeline.
|
|
|
|
Steps:
|
|
1. Download + extract numpy source sdist from PyPI
|
|
2. Cross-compile with vendored meson + OHOS cross file
|
|
3. Assemble wheel from Python source + cross-compiled .so files
|
|
"""
|
|
if not _check_toolchain():
|
|
return None
|
|
|
|
out_dir = output_dir or OUTPUT_DIR
|
|
sources = _ensure_dirs(SOURCES_DIR)
|
|
|
|
# Download + extract
|
|
tarball = _download_sdist("numpy", version, sources)
|
|
src_dir = _extract_tarball(tarball, sources)
|
|
|
|
build_dir = os.path.join(sources, f"numpy-{version}-build")
|
|
if os.path.isdir(build_dir) and not rebuild:
|
|
print(f"[rebuild] Build dir already exists: {build_dir}")
|
|
print("[rebuild] Set rebuild=True or remove the dir to rebuild from scratch")
|
|
else:
|
|
ok, build_dir = cross_compile_numpy(src_dir, build_dir)
|
|
if not ok:
|
|
print("[rebuild] Cross-compilation FAILED")
|
|
return None
|
|
|
|
# Assemble wheel
|
|
wheel_path = assemble_wheel(src_dir, build_dir, out_dir,
|
|
pkg_name="numpy", version=version,
|
|
target_os="harmonyos", arch="arm64")
|
|
return wheel_path
|
|
|
|
|
|
def rebuild_from_source(source_dir, pkg_name, output_dir, version="0.0.0",
|
|
target_os="harmonyos", arch="arm64"):
|
|
"""Generic entry point: rebuild a source package for HarmonyOS.
|
|
|
|
For now, dispatches to package-specific builders.
|
|
Falls back to stub wheel generation for unknown packages.
|
|
"""
|
|
if pkg_name == "numpy":
|
|
# Use default numpy version if no real version detected (e.g. git clone failed)
|
|
if version == "0.0.0":
|
|
version = "1.26.4"
|
|
return build_numpy_wheel(version=version, output_dir=output_dir)
|
|
return _stub_wheel(source_dir, pkg_name, output_dir, version, target_os, arch)
|
|
|
|
|
|
def rebuild_native_wheel(wheel_path: str, output_dir: str,
|
|
target_os: str = "harmonyos", arch: str = "arm64") -> str:
|
|
"""Entry point for wheel-to-wheel rebuild (not yet implemented)."""
|
|
print(f"[rebuild] Native wheel rebuild from wheel not yet supported: {wheel_path}")
|
|
print("[rebuild] Use --manifest or rebuild_from_source() instead.")
|
|
basename = os.path.basename(wheel_path)
|
|
output_path = os.path.join(output_dir, basename)
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
shutil.copy2(wheel_path, output_path)
|
|
return output_path
|
|
|
|
|
|
def _stub_wheel(source_dir, pkg_name, output_dir, version, target_os, arch):
|
|
"""Generate a stub wheel with metadata only."""
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
wheel_name = f"{pkg_name}-{version}-cp312-cp312-{target_os}_{arch}.whl"
|
|
wheel_path = os.path.join(output_dir, wheel_name)
|
|
dist_info = f"{pkg_name}-{version}.dist-info"
|
|
with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
zf.writestr(f"{dist_info}/METADATA",
|
|
f"Metadata-Version: 2.1\nName: {pkg_name}\nVersion: {version}\n"
|
|
f"Summary: EcoArk HarmonyOS rebuild (stub)\n")
|
|
zf.writestr(f"{dist_info}/WHEEL",
|
|
f"Wheel-Version: 1.0\nGenerator: ecoark-pipeline\n"
|
|
f"Root-Is-Purelib: false\nTag: cp312-cp312-{target_os}_{arch}\n")
|
|
zf.writestr(f"{dist_info}/RECORD", "")
|
|
zf.writestr(f"{pkg_name}/__init__.py",
|
|
f"def hello(): return 'built by EcoArk pipeline (stub)'\n")
|
|
zf.writestr(f"{pkg_name}/_ecoark_stub.txt",
|
|
f"Package: {pkg_name}\nSource: {source_dir}\n"
|
|
f"Target: {target_os}/{arch}\nStub wheel (replace with real build).\n")
|
|
print(f"[rebuild] Stub wheel: {wheel_path}")
|
|
return wheel_path
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
if len(sys.argv) > 1 and sys.argv[1] == "numpy":
|
|
build_numpy_wheel(version=sys.argv[2] if len(sys.argv) > 2 else "1.26.4")
|
|
else:
|
|
print("Usage: python -m pipeline.rebuild numpy [version]")
|
|
print(" python -m pipeline.rebuild numpy 1.26.4")
|