setup.py

import importlib
import json
import os
import platform
import shutil
import subprocess
import sys
import sysconfig
from pathlib import Path

from setuptools import Distribution, Extension, setup
from setuptools.command.build_ext import build_ext
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist

try:
    from wheel.bdist_wheel import bdist_wheel
except ImportError:  # pragma: no cover - local build_ext may not have wheel installed
    bdist_wheel = None

SETUP_DIR = Path(__file__).resolve().parent
if str(SETUP_DIR) not in sys.path:
    sys.path.insert(0, str(SETUP_DIR))

get_host_engine_build_config = importlib.import_module(
    "scripts.build_support.x86_profiles"
).get_host_engine_build_config
resolve_openviking_version = importlib.import_module(
    "scripts.build_support.versioning"
).resolve_openviking_version
pypi_long_description = importlib.import_module(
    "scripts.build_support.readme"
).pypi_long_description
write_system_assets_manifest = importlib.import_module(
    "scripts.build_support.system_assets_manifest"
).write_system_assets_manifest

CMAKE_PATH = shutil.which("cmake") or "cmake"
C_COMPILER_PATH = os.environ.get("CC") or shutil.which("gcc") or "gcc"
CXX_COMPILER_PATH = os.environ.get("CXX") or shutil.which("g++") or "g++"
ENGINE_SOURCE_DIR = "src/"
ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())
SKIP_CPP_BUILD = os.environ.get("OV_SKIP_CPP_BUILD") == "1"


def _sanitize_native_build_env(env):
    """Keep Rust native builds from accidentally linking against Linuxbrew libs.

    On older glibc systems, Homebrew-provided native libraries can require a newer
    libc than the host linker/runtime supports. When pkg-config resolves xz/bzip2
    from Linuxbrew, Cargo inherits those library search paths and link fails.
    """

    sanitized_env = env.copy()

    pkg_config = sanitized_env.get("PKG_CONFIG") or shutil.which("pkg-config")
    if pkg_config and "linuxbrew" in os.path.realpath(pkg_config).lower():
        system_pkg_config = "/usr/bin/pkg-config"
        if Path(system_pkg_config).exists():
            sanitized_env["PKG_CONFIG"] = system_pkg_config

    for key in ("PKG_CONFIG_PATH", "LIBRARY_PATH", "LD_LIBRARY_PATH"):
        value = sanitized_env.get(key)
        if not value:
            continue
        kept_paths = [
            path
            for path in value.split(os.pathsep)
            if path and "linuxbrew" not in os.path.realpath(path).lower()
        ]
        if kept_paths:
            sanitized_env[key] = os.pathsep.join(kept_paths)
        else:
            sanitized_env.pop(key, None)

    return sanitized_env


def _get_windows_python_sabi_library() -> Path:
    """Return the stable-ABI Python library path for Windows abi3 extensions."""
    candidate_roots = []
    for raw_root in (
        sys.base_prefix,
        sys.base_exec_prefix,
        sysconfig.get_config_var("installed_base"),
        sysconfig.get_config_var("base"),
    ):
        if not raw_root:
            continue
        candidate_root = Path(raw_root).resolve()
        if candidate_root not in candidate_roots:
            candidate_roots.append(candidate_root)

    candidate_paths = []
    for root in candidate_roots:
        candidate_paths.extend(
            [
                root / "libs" / "python3.lib",
                root / "python3.dll",
            ]
        )

    for candidate_path in candidate_paths:
        if candidate_path.exists():
            return candidate_path

    searched = ", ".join(str(path) for path in candidate_paths) or "<none>"
    raise RuntimeError(
        "Could not locate the Windows stable-ABI Python library for abi3 engine modules. "
        f"Searched: {searched}"
    )


class OpenVikingBuildExt(build_ext):
    """Build OpenViking runtime artifacts and Python native extensions."""

    def run(self):
        self.build_ov_cli_artifact()
        self.build_ragfs_python_artifact()
        self.cmake_executable = CMAKE_PATH

        if SKIP_CPP_BUILD:
            print("[SKIP] C++ vector engine build disabled by OV_SKIP_CPP_BUILD=1")
        else:
            for ext in self.extensions:
                self.build_extension(ext)

    def _copy_artifact(self, src, dst):
        """Copy a build artifact into the package tree and preserve executability."""
        print(f"Copying artifact from {src} to {dst}")
        dst.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(str(src), str(dst))
        if sys.platform != "win32":
            os.chmod(str(dst), 0o755)

    def _copy_artifacts_to_build_lib(self, target_binary=None, target_lib=None):
        """Copy built artifacts into build_lib so wheel packaging can include them."""
        if self.build_lib:
            build_pkg_dir = Path(self.build_lib) / "openviking"
            if target_binary and target_binary.exists():
                self._copy_artifact(target_binary, build_pkg_dir / "bin" / target_binary.name)
            if target_lib and target_lib.exists():
                self._copy_artifact(target_lib, build_pkg_dir / "lib" / target_lib.name)

    def _require_artifact(self, artifact_path, artifact_name, stage_name):
        """Abort the build immediately when a required artifact is missing."""
        if artifact_path.exists():
            return
        raise RuntimeError(
            f"{stage_name} did not produce required {artifact_name} at {artifact_path}"
        )

    def _run_stage_with_artifact_checks(
        self, stage_name, build_fn, required_artifacts, on_success=None
    ):
        """Run a build stage and always validate its required outputs on normal return."""
        build_fn()
        for artifact_path, artifact_name in required_artifacts:
            self._require_artifact(artifact_path, artifact_name, stage_name)
        if on_success:
            on_success()

    def _resolve_cargo_target_dir(self, cargo_project_dir, env):
        """Resolve the Cargo target directory for workspace and overridden builds."""
        configured_target_dir = env.get("CARGO_TARGET_DIR")
        if configured_target_dir:
            return Path(configured_target_dir).resolve()

        try:
            result = subprocess.run(
                ["cargo", "metadata", "--format-version", "1", "--no-deps"],
                cwd=str(cargo_project_dir),
                env=env,
                check=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            metadata = json.loads(result.stdout.decode("utf-8"))
            target_directory = metadata.get("target_directory")
            if target_directory:
                return Path(target_directory).resolve()
        except Exception as exc:
            print(f"[Warning] Failed to resolve Cargo target directory via metadata: {exc}")

        return cargo_project_dir.parents[1] / "target"

    def build_ov_cli_artifact(self):
        """Build or reuse the ov Rust CLI binary."""
        if os.environ.get("OV_SKIP_OV_BUILD") == "1":
            print("[SKIP] ov CLI build disabled by OV_SKIP_OV_BUILD=1")
            return

        binary_name = "ov.exe" if sys.platform == "win32" else "ov"
        ov_cli_dir = Path("crates/ov_cli").resolve()
        ov_target_binary = Path("openviking/bin").resolve() / binary_name

        self._run_stage_with_artifact_checks(
            "ov CLI build",
            lambda: self._build_ov_cli_artifact_impl(ov_cli_dir, binary_name, ov_target_binary),
            [(ov_target_binary, binary_name)],
            on_success=lambda: self._copy_artifacts_to_build_lib(ov_target_binary, None),
        )

    def _build_ov_cli_artifact_impl(self, ov_cli_dir, binary_name, ov_target_binary):
        """Implement ov CLI building without final artifact checks."""

        prebuilt_dir = os.environ.get("OV_PREBUILT_BIN_DIR")
        if prebuilt_dir:
            src_bin = Path(prebuilt_dir).resolve() / binary_name
            if src_bin.exists():
                self._copy_artifact(src_bin, ov_target_binary)
                return

        if ov_cli_dir.exists() and shutil.which("cargo"):
            print("Building ov CLI from source...")
            try:
                env = _sanitize_native_build_env(os.environ.copy())
                env["OPENVIKING_VERSION"] = resolve_openviking_version(
                    env=env, project_root=SETUP_DIR
                )
                build_args = ["cargo", "build", "--release"]
                target = env.get("CARGO_BUILD_TARGET")
                if target:
                    print(f"Cross-compiling with CARGO_BUILD_TARGET={target}")
                    build_args.extend(["--target", target])

                result = subprocess.run(
                    build_args,
                    cwd=str(ov_cli_dir),
                    env=env,
                    check=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
                if result.stdout:
                    print(f"Build stdout: {result.stdout.decode('utf-8', errors='replace')}")
                if result.stderr:
                    print(f"Build stderr: {result.stderr.decode('utf-8', errors='replace')}")

                cargo_target_dir = self._resolve_cargo_target_dir(ov_cli_dir, env)
                if target:
                    built_bin = cargo_target_dir / target / "release" / binary_name
                else:
                    built_bin = cargo_target_dir / "release" / binary_name

                self._require_artifact(built_bin, binary_name, "ov CLI build")
                self._copy_artifact(built_bin, ov_target_binary)
                print("[OK] ov CLI built successfully from source")
            except Exception as exc:
                error_msg = f"Failed to build ov CLI from source: {exc}"
                if isinstance(exc, subprocess.CalledProcessError):
                    if exc.stdout:
                        error_msg += (
                            f"\nBuild stdout:\n{exc.stdout.decode('utf-8', errors='replace')}"
                        )
                    if exc.stderr:
                        error_msg += (
                            f"\nBuild stderr:\n{exc.stderr.decode('utf-8', errors='replace')}"
                        )
                print(f"[Error] {error_msg}")
                raise RuntimeError(error_msg)
        else:
            if ov_target_binary.exists():
                print("[Info] ov CLI binary already exists locally. Skipping source build.")
            elif not ov_cli_dir.exists():
                print(f"[Warning] ov CLI source directory not found at {ov_cli_dir}")
            else:
                print("[Warning] Cargo not found. Cannot build ov CLI from source.")

    def build_ragfs_python_artifact(self):
        """Build ragfs-python (Rust RAGFS binding) via maturin and copy the native
        extension into ``openviking/lib/`` so it ships inside the openviking wheel.
        """
        require_ragfs_artifact = self._should_require_ragfs_artifact()
        ragfs_python_dir = Path("crates/ragfs-python").resolve()
        ragfs_lib_dir = Path("openviking/lib").resolve()

        if not ragfs_python_dir.exists():
            message = "ragfs-python source directory not found."
            if require_ragfs_artifact:
                raise RuntimeError(message)
            print(f"[Info] {message} Skipping.")
            return

        if os.environ.get("OV_SKIP_RAGFS_BUILD") == "1":
            message = "Skipping ragfs-python build (OV_SKIP_RAGFS_BUILD=1)"
            if require_ragfs_artifact:
                raise RuntimeError(f"{message} is incompatible with required wheel artifacts.")
            print(f"[OK] {message}")
            return

        if importlib.util.find_spec("maturin") is None:
            message = (
                "maturin not found. ragfs-python (Rust binding) will not be built.\n"
                "       Install maturin to enable: pip install maturin"
            )
            if require_ragfs_artifact:
                raise RuntimeError(message)
            print(f"[SKIP] {message}")
            return

        import tempfile
        import zipfile

        with tempfile.TemporaryDirectory() as tmpdir:
            try:
                print("Building ragfs-python (Rust RAGFS binding) via maturin...")
                env = _sanitize_native_build_env(os.environ.copy())
                build_args = [
                    sys.executable,
                    "-m",
                    "maturin",
                    "build",
                    "--release",
                    "--out",
                    tmpdir,
                ]
                # Respect CARGO_BUILD_TARGET for cross-compilation
                target = env.get("CARGO_BUILD_TARGET")
                if target:
                    build_args.extend(["--target", target])

                result = subprocess.run(
                    build_args,
                    cwd=str(ragfs_python_dir),
                    env=env,
                    check=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
                if result.stdout:
                    print(result.stdout.decode("utf-8", errors="replace"))
                if result.stderr:
                    print(result.stderr.decode("utf-8", errors="replace"))

                # Extract the stable-ABI native extension from the built wheel.
                whl_files = list(Path(tmpdir).glob("ragfs_python-*.whl"))
                if not whl_files:
                    message = "maturin produced no wheel for ragfs-python."
                    if require_ragfs_artifact:
                        raise RuntimeError(message)
                    print(f"[Warning] {message}")
                    return

                ragfs_lib_dir.mkdir(parents=True, exist_ok=True)
                for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.so"):
                    stale_artifact.unlink()
                for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.pyd"):
                    stale_artifact.unlink()
                for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.dylib"):
                    stale_artifact.unlink()

                extracted = False
                with zipfile.ZipFile(str(whl_files[0])) as zf:
                    for name in zf.namelist():
                        basename = Path(name).name
                        is_ragfs_extension = basename == "ragfs_python.pyd" or (
                            basename.startswith("ragfs_python.abi3.")
                            and basename.endswith((".so", ".pyd"))
                        )
                        if is_ragfs_extension:
                            target_path = ragfs_lib_dir / basename
                            with zf.open(name) as src, open(target_path, "wb") as dst:
                                dst.write(src.read())
                            if sys.platform != "win32":
                                os.chmod(str(target_path), 0o755)
                            print(f"[OK] ragfs-python: extracted {basename} -> {target_path}")
                            extracted = True
                            break

                if not extracted:
                    message = (
                        "Could not find ragfs_python stable-ABI native extension in built wheel."
                    )
                    if require_ragfs_artifact:
                        raise RuntimeError(message)
                    print(f"[Warning] {message}")
                else:
                    self._copy_artifacts_to_build_lib(target_lib=target_path)

            except Exception as exc:
                error_detail = ""
                if isinstance(exc, subprocess.CalledProcessError):
                    if exc.stdout:
                        error_detail += exc.stdout.decode("utf-8", errors="replace")
                    if exc.stderr:
                        error_detail += exc.stderr.decode("utf-8", errors="replace")
                if require_ragfs_artifact:
                    error_message = f"Failed to build ragfs-python: {exc}"
                    if error_detail:
                        error_message += f"\n{error_detail}"
                    raise RuntimeError(error_message) from exc
                print(f"[Warning] Failed to build ragfs-python: {exc}")
                if error_detail:
                    print(error_detail)

    def _should_require_ragfs_artifact(self) -> bool:
        """Fail wheel builds closed when ragfs-python cannot be bundled."""
        required = os.environ.get("OV_REQUIRE_RAGFS_BUILD")
        if required is not None:
            return required == "1"
        return "bdist_wheel" in sys.argv

    def build_extension(self, ext):
        """Build a single Python native extension artifact using CMake."""
        if getattr(self, "_engine_extensions_built", False):
            return

        ext_fullpath = Path(self.get_ext_fullpath(ext.name))
        ext_dir = ext_fullpath.parent.resolve()
        build_dir = Path(self.build_temp) / "cmake_build"
        build_dir.mkdir(parents=True, exist_ok=True)
        self._clean_stale_engine_artifacts(ext_dir)

        self._run_stage_with_artifact_checks(
            "CMake build",
            lambda: self._build_extension_impl(ext_fullpath, ext_dir, build_dir),
            [(ext_fullpath, f"native extension '{ext.name}'")],
        )
        self._engine_extensions_built = True

    def _clean_stale_engine_artifacts(self, ext_dir: Path):
        """Remove stale non-abi3 engine binaries from wheel build output directories."""
        source_engine_dir = (SETUP_DIR / "openviking" / "storage" / "vectordb" / "engine").resolve()
        if ext_dir == source_engine_dir:
            return

        for pattern in ("*.so", "*.pyd"):
            for artifact in ext_dir.glob(pattern):
                artifact.unlink()

    def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
        """Invoke CMake to build the Python native extension."""
        ext_basename = ext_fullpath.stem.split(".")[0]
        built_filename = Path(self.get_ext_filename(self.extensions[0].name)).name
        py_ext_suffix = built_filename.removeprefix(ext_basename)
        if not py_ext_suffix:
            py_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ext_fullpath.suffix

        cmake_args = [
            f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",
            f"-B{build_dir}",
            "-DCMAKE_BUILD_TYPE=Release",
            f"-DOV_PY_OUTPUT_DIR={ext_dir}",
            f"-DOV_PY_EXT_SUFFIX={py_ext_suffix}",
            f"-DOV_X86_BUILD_VARIANTS={';'.join(ENGINE_BUILD_CONFIG.cmake_variants)}",
            "-DCMAKE_VERBOSE_MAKEFILE=ON",
            "-DCMAKE_INSTALL_RPATH=$ORIGIN",
            f"-DPython3_EXECUTABLE={sys.executable}",
            f"-DPython3_INCLUDE_DIRS={sysconfig.get_path('include')}",
            f"-DPython3_LIBRARIES={sysconfig.get_config_vars().get('LIBRARY')}",
            f"-DCMAKE_C_COMPILER={C_COMPILER_PATH}",
            f"-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}",
        ]

        if sys.platform == "darwin":
            cmake_args.append("-DCMAKE_OSX_DEPLOYMENT_TARGET=10.15")
            target_arch = os.environ.get("CMAKE_OSX_ARCHITECTURES")
            if target_arch:
                cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}")
        elif sys.platform == "win32":
            windows_python_sabi_library = _get_windows_python_sabi_library()
            cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}")
            cmake_args.extend(["-G", "MinGW Makefiles"])

        self.spawn([self.cmake_executable] + cmake_args)

        build_args = ["--build", str(build_dir), "--config", "Release", f"-j{os.cpu_count() or 4}"]
        self.spawn([self.cmake_executable] + build_args)


def _build_web_studio():
    """Build the web-studio SPA and copy dist into the Python package tree.

    Skipped when OV_SKIP_STUDIO_BUILD=1 or when the bundle already exists.
    Falls back gracefully (warning, not error) when npm is unavailable.
    """
    if os.environ.get("OV_SKIP_STUDIO_BUILD") == "1":
        print("  [SKIP] web-studio build disabled by OV_SKIP_STUDIO_BUILD=1")
        return

    dest = SETUP_DIR / "openviking" / "web_studio" / "dist"
    if (dest / "index.html").is_file():
        print("  [OK] web-studio bundle already present")
        return

    source = SETUP_DIR / "web-studio"
    if not (source / "package.json").is_file():
        print("  [SKIP] web-studio source not found; /studio will be unavailable")
        return

    npm = shutil.which("npm")
    if not npm:
        print("  [SKIP] npm not found; install Node.js to enable /studio")
        return

    print("Building web-studio (Vite SPA)...")
    try:
        subprocess.check_call([npm, "ci"], cwd=str(source))
        subprocess.check_call(
            [npm, "run", "build", "--", "--base=/studio/"],
            cwd=str(source),
        )
    except subprocess.CalledProcessError as exc:
        print(f"  [WARNING] web-studio npm build failed ({exc}); /studio will be unavailable")
        return

    built = source / "dist"
    if not (built / "index.html").is_file():
        print("  [WARNING] web-studio build produced no index.html; /studio will be unavailable")
        return

    dest.parent.mkdir(parents=True, exist_ok=True)
    if dest.exists():
        shutil.rmtree(dest)
    shutil.copytree(built, dest)
    print(f"  [OK] web-studio bundle copied to {dest}")


class OpenVikingBuildPy(build_py):
    def run(self):
        _build_web_studio()
        write_system_assets_manifest(SETUP_DIR / "openviking" / "system_assets")
        super().run()
        package_root = Path(self.build_lib) / "vikingbot"
        for asset_name in ("workspace", "bridge"):
            source = SETUP_DIR / "bot" / asset_name
            if not source.is_dir():
                raise FileNotFoundError(f"required VikingBot asset directory not found: {source}")
            destination = package_root / asset_name
            if destination.exists():
                shutil.rmtree(destination)
            shutil.copytree(
                source,
                destination,
                ignore=shutil.ignore_patterns("node_modules", "dist", "__pycache__", "*.pyc"),
            )


class OpenVikingSdist(sdist):
    def make_release_tree(self, base_dir, files):
        super().make_release_tree(base_dir, files)
        write_system_assets_manifest(Path(base_dir) / "openviking" / "system_assets")


if bdist_wheel is not None:

    class OpenVikingBdistWheel(bdist_wheel):
        def finalize_options(self):
            super().finalize_options()
            self.py_limited_api = "cp310"
else:
    OpenVikingBdistWheel = None


class OpenVikingDistribution(Distribution):
    """Keep wheels platform-specific because ragfs-python is a native artifact."""

    def has_ext_modules(self):
        return True


cmdclass = {
    "build_ext": OpenVikingBuildExt,
    "build_py": OpenVikingBuildPy,
    "sdist": OpenVikingSdist,
}
if OpenVikingBdistWheel is not None:
    cmdclass["bdist_wheel"] = OpenVikingBdistWheel


setup(
    distclass=OpenVikingDistribution,
    long_description=pypi_long_description(SETUP_DIR / "README.md"),
    long_description_content_type="text/markdown",
    ext_modules=(
        []
        if SKIP_CPP_BUILD
        else [
            Extension(
                name=ENGINE_BUILD_CONFIG.primary_extension,
                sources=[],
                py_limited_api=True,
            )
        ]
    ),
    cmdclass=cmdclass,
    package_data={
        "openviking_gateway": ["client-rules.json"],
        "openviking.scheduling.expiry": ["lua/*.lua"],
        "openviking": [
            "lib/ragfs_python*.so",
            "lib/ragfs_python*.pyd",
            "bin/ov",
            "bin/ov.exe",
            "server/account_stores/sql/*.sql",
            "server/account_stores/sql/changes/*.sql",
            "service/sql/*.sql",
            "service/sql/changes/*.sql",
            "server/static/**/*",
            "web_studio/dist/**/*",
            "storage/vectordb/engine/*.abi3.so",
            "storage/vectordb/engine/*.pyd",
        ],
        "vikingbot": [
            "workspace/**/*",
            "bridge/**/*",
        ],
    },
    include_package_data=True,
)