Add Python bindings build using bzlmod (#1764)

* Add a bzlmod Python bindings build

Uses the newly started `@nanobind_bazel` project to build nanobind
extensions. This means that we can drop all in-tree custom build defs
and build files for nanobind and the C++ Python headers.

Additionally, the temporary WORKSPACE overwrite hack naturally goes away
due to the WORKSPACE system being obsolete.

* Bump ruff -> v0.3.1, change ruff settings

The latest minor releases incurred some formatting and configuration
changes, this commit rolls them out.

---------

Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
         types_or: [ python, pyi ]
         args: [ "--ignore-missing-imports", "--scripts-are-modules" ]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.13
+    rev: v0.3.1
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]

MODULE.bazel

@@ -4,11 +4,11 @@ module(
 )
 
 bazel_dep(name = "bazel_skylib", version = "1.5.0")
-bazel_dep(name = "platforms", version = "0.0.7")
+bazel_dep(name = "platforms", version = "0.0.8")
 bazel_dep(name = "rules_foreign_cc", version = "0.10.1")
 bazel_dep(name = "rules_cc", version = "0.0.9")
 
-bazel_dep(name = "rules_python", version = "0.27.1", dev_dependency = True)
+bazel_dep(name = "rules_python", version = "0.31.0", dev_dependency = True)
 bazel_dep(name = "googletest", version = "1.12.1", dev_dependency = True, repo_name = "com_google_googletest")
 
 bazel_dep(name = "libpfm", version = "4.11.0")
@@ -19,7 +19,18 @@ bazel_dep(name = "libpfm", version = "4.11.0")
 # of relying on the changing default version from rules_python.
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True)
+python.toolchain(python_version = "3.8")
 python.toolchain(python_version = "3.9")
+python.toolchain(python_version = "3.10")
+python.toolchain(python_version = "3.11")
+python.toolchain(
+    is_default = True,
+    python_version = "3.12",
+)
+use_repo(
+    python,
+    python = "python_versions",
+)
 
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True)
 pip.parse(
@@ -30,3 +41,10 @@ pip.parse(
 use_repo(pip, "tools_pip_deps")
 
 # -- bazel_dep definitions -- #
+
+bazel_dep(name = "nanobind_bazel", version = "", dev_dependency = True)
+git_override(
+    module_name = "nanobind_bazel",
+    commit = "97e3db2744d3f5da244a0846a0644ffb074b4880",
+    remote = "https://github.com/nicholasjng/nanobind-bazel",
+)

WORKSPACE

@@ -22,9 +22,3 @@ pip_parse(
 load("@tools_pip_deps//:requirements.bzl", "install_deps")
 
 install_deps()
-
-new_local_repository(
-    name = "python_headers",
-    build_file = "@//bindings/python:python_headers.BUILD",
-    path = "<PYTHON_INCLUDE_PATH>",  # May be overwritten by setup.py.
-)

bindings/python/BUILD

@@ -1,3 +0,0 @@
-exports_files(glob(["*.BUILD"]))
-
-exports_files(["build_defs.bzl"])

bindings/python/build_defs.bzl

@@ -1,29 +0,0 @@
-"""
-This file contains some build definitions for C++ extensions used in the Google Benchmark Python bindings.
-"""
-
-_SHARED_LIB_SUFFIX = {
-    "//conditions:default": ".so",
-    "//:windows": ".dll",
-}
-
-def py_extension(name, srcs, hdrs = [], copts = [], features = [], deps = []):
-    for shared_lib_suffix in _SHARED_LIB_SUFFIX.values():
-        shared_lib_name = name + shared_lib_suffix
-        native.cc_binary(
-            name = shared_lib_name,
-            linkshared = True,
-            linkstatic = True,
-            srcs = srcs + hdrs,
-            copts = copts,
-            features = features,
-            deps = deps,
-        )
-
-    return native.py_library(
-        name = name,
-        data = select({
-            platform: [name + shared_lib_suffix]
-            for platform, shared_lib_suffix in _SHARED_LIB_SUFFIX.items()
-        }),
-    )

bindings/python/google_benchmark/BUILD

@@ -1,4 +1,4 @@
-load("//bindings/python:build_defs.bzl", "py_extension")
+load("@nanobind_bazel//:build_defs.bzl", "nanobind_extension")
 
 py_library(
     name = "google_benchmark",
@@ -9,22 +9,10 @@ py_library(
     ],
 )
 
-py_extension(
+nanobind_extension(
     name = "_benchmark",
     srcs = ["benchmark.cc"],
-    copts = [
+    deps = ["//:benchmark"],
-        "-fexceptions",
-        "-fno-strict-aliasing",
-    ],
-    features = [
-        "-use_header_modules",
-        "-parse_headers",
-    ],
-    deps = [
-        "//:benchmark",
-        "@nanobind",
-        "@python_headers",
-    ],
 )
 
 py_test(

bindings/python/google_benchmark/__init__.py

@@ -26,6 +26,7 @@ Example usage:
   if __name__ == '__main__':
     benchmark.main()
 """
+
 import atexit
 
 from absl import app

bindings/python/nanobind.BUILD

@@ -1,59 +0,0 @@
-load("@bazel_skylib//lib:selects.bzl", "selects")
-
-licenses(["notice"])
-
-package(default_visibility = ["//visibility:public"])
-
-config_setting(
-    name = "msvc_compiler",
-    flag_values = {"@bazel_tools//tools/cpp:compiler": "msvc-cl"},
-)
-
-selects.config_setting_group(
-    name = "winplusmsvc",
-    match_all = [
-        "@platforms//os:windows",
-        ":msvc_compiler",
-    ],
-)
-
-cc_library(
-    name = "nanobind",
-    srcs = glob([
-        "src/*.cpp",
-    ]),
-    additional_linker_inputs = select({
-        "@platforms//os:macos": [":cmake/darwin-ld-cpython.sym"],
-        "//conditions:default": [],
-    }),
-    copts = select({
-        ":msvc_compiler": [
-            "/EHsc",  # exceptions
-            "/Os",  # size optimizations
-            "/GL",  # LTO / whole program optimization
-        ],
-        # these should work on both clang and gcc.
-        "//conditions:default": [
-            "-fexceptions",
-            "-flto",
-            "-Os",
-        ],
-    }),
-    includes = [
-        "ext/robin_map/include",
-        "include",
-    ],
-    linkopts = select({
-        ":winplusmsvc": ["/LTGC"],  # Windows + MSVC.
-        "@platforms//os:macos": ["-Wl,@$(location :cmake/darwin-ld-cpython.sym)"],  # Apple.
-        "//conditions:default": [],
-    }),
-    textual_hdrs = glob(
-        [
-            "include/**/*.h",
-            "src/*.h",
-            "ext/robin_map/include/tsl/*.h",
-        ],
-    ),
-    deps = ["@python_headers"],
-)

bindings/python/python_headers.BUILD

@@ -1,10 +0,0 @@
-licenses(["notice"])
-
-package(default_visibility = ["//visibility:public"])
-
-cc_library(
-    name = "python_headers",
-    hdrs = glob(["**/*.h"]),
-    includes = ["."],
-    visibility = ["//visibility:public"],
-)

pyproject.toml

@@ -75,11 +75,12 @@ src = ["bindings/python"]
 line-length = 80
 target-version = "py311"
 
+[tool.ruff.lint]
 # Enable pycodestyle (`E`, `W`), Pyflakes (`F`), and isort (`I`) codes by default.
 select = ["E", "F", "I", "W"]
 ignore = [
     "E501", # line too long
 ]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 combine-as-imports = true

setup.py

@@ -1,46 +1,27 @@
-import contextlib
 import os
 import platform
 import shutil
-import sysconfig
 from pathlib import Path
-from typing import Generator
+from typing import Any
 
 import setuptools
 from setuptools.command import build_ext
 
-PYTHON_INCLUDE_PATH_PLACEHOLDER = "<PYTHON_INCLUDE_PATH>"
-
 IS_WINDOWS = platform.system() == "Windows"
 IS_MAC = platform.system() == "Darwin"
 
-
+# hardcoded SABI-related options. Requires that each Python interpreter
-@contextlib.contextmanager
+# (hermetic or not) participating is of the same major-minor version.
-def temp_fill_include_path(fp: str) -> Generator[None, None, None]:
+version_tuple = tuple(int(i) for i in platform.python_version_tuple())
-    """Temporarily set the Python include path in a file."""
+py_limited_api = version_tuple >= (3, 12)
-    with open(fp, "r+") as f:
+options = {"bdist_wheel": {"py_limited_api": "cp312"}} if py_limited_api else {}
-        try:
-            content = f.read()
-            replaced = content.replace(
-                PYTHON_INCLUDE_PATH_PLACEHOLDER,
-                Path(sysconfig.get_paths()["include"]).as_posix(),
-            )
-            f.seek(0)
-            f.write(replaced)
-            f.truncate()
-            yield
-        finally:
-            # revert to the original content after exit
-            f.seek(0)
-            f.write(content)
-            f.truncate()
 
 
 class BazelExtension(setuptools.Extension):
     """A C/C++ extension that is defined as a Bazel BUILD target."""
 
-    def __init__(self, name: str, bazel_target: str):
+    def __init__(self, name: str, bazel_target: str, **kwargs: Any):
-        super().__init__(name=name, sources=[])
+        super().__init__(name=name, sources=[], **kwargs)
 
         self.bazel_target = bazel_target
         stripped_target = bazel_target.split("//")[-1]
@@ -67,49 +48,58 @@ class BuildBazelExtension(build_ext.build_ext):
 
     def bazel_build(self, ext: BazelExtension) -> None:
         """Runs the bazel build to create the package."""
-        with temp_fill_include_path("WORKSPACE"):
+        temp_path = Path(self.build_temp)
-            temp_path = Path(self.build_temp)
+        # omit the patch version to avoid build errors if the toolchain is not
+        # yet registered in the current @rules_python version.
+        # patch version differences should be fine.
+        python_version = ".".join(platform.python_version_tuple()[:2])
 
-            bazel_argv = [
+        bazel_argv = [
-                "bazel",
+            "bazel",
-                "build",
+            "build",
-                ext.bazel_target,
+            ext.bazel_target,
-                "--enable_bzlmod=false",
+            f"--symlink_prefix={temp_path / 'bazel-'}",
-                f"--symlink_prefix={temp_path / 'bazel-'}",
+            f"--compilation_mode={'dbg' if self.debug else 'opt'}",
-                f"--compilation_mode={'dbg' if self.debug else 'opt'}",
+            # C++17 is required by nanobind
-                # C++17 is required by nanobind
+            f"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}",
-                f"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}",
+            f"--@rules_python//python/config_settings:python_version={python_version}",
-            ]
+        ]
 
-            if IS_WINDOWS:
+        if ext.py_limited_api:
-                # Link with python*.lib.
+            bazel_argv += ["--@nanobind_bazel//:py-limited-api=cp312"]
-                for library_dir in self.library_dirs:
-                    bazel_argv.append("--linkopt=/LIBPATH:" + library_dir)
-            elif IS_MAC:
-                if platform.machine() == "x86_64":
-                    # C++17 needs macOS 10.14 at minimum
-                    bazel_argv.append("--macos_minimum_os=10.14")
 
-                    # cross-compilation for Mac ARM64 on GitHub Mac x86 runners.
+        if IS_WINDOWS:
-                    # ARCHFLAGS is set by cibuildwheel before macOS wheel builds.
+            # Link with python*.lib.
-                    archflags = os.getenv("ARCHFLAGS", "")
+            for library_dir in self.library_dirs:
-                    if "arm64" in archflags:
+                bazel_argv.append("--linkopt=/LIBPATH:" + library_dir)
-                        bazel_argv.append("--cpu=darwin_arm64")
+        elif IS_MAC:
-                        bazel_argv.append("--macos_cpus=arm64")
+            if platform.machine() == "x86_64":
+                # C++17 needs macOS 10.14 at minimum
+                bazel_argv.append("--macos_minimum_os=10.14")
 
-                elif platform.machine() == "arm64":
+                # cross-compilation for Mac ARM64 on GitHub Mac x86 runners.
-                    bazel_argv.append("--macos_minimum_os=11.0")
+                # ARCHFLAGS is set by cibuildwheel before macOS wheel builds.
+                archflags = os.getenv("ARCHFLAGS", "")
+                if "arm64" in archflags:
+                    bazel_argv.append("--cpu=darwin_arm64")
+                    bazel_argv.append("--macos_cpus=arm64")
 
-            self.spawn(bazel_argv)
+            elif platform.machine() == "arm64":
+                bazel_argv.append("--macos_minimum_os=11.0")
 
-            shared_lib_suffix = ".dll" if IS_WINDOWS else ".so"
+        self.spawn(bazel_argv)
-            ext_name = ext.target_name + shared_lib_suffix
-            ext_bazel_bin_path = (
-                temp_path / "bazel-bin" / ext.relpath / ext_name
-            )
 
-            ext_dest_path = Path(self.get_ext_fullpath(ext.name))
+        if IS_WINDOWS:
-            shutil.copyfile(ext_bazel_bin_path, ext_dest_path)
+            suffix = ".pyd"
+        else:
+            suffix = ".abi3.so" if ext.py_limited_api else ".so"
+
+        ext_name = ext.target_name + suffix
+        ext_bazel_bin_path = temp_path / "bazel-bin" / ext.relpath / ext_name
+        ext_dest_path = Path(self.get_ext_fullpath(ext.name)).with_name(
+            ext_name
+        )
+        shutil.copyfile(ext_bazel_bin_path, ext_dest_path)
 
 
 setuptools.setup(
@@ -118,6 +108,8 @@ setuptools.setup(
         BazelExtension(
             name="google_benchmark._benchmark",
             bazel_target="//bindings/python/google_benchmark:_benchmark",
+            py_limited_api=py_limited_api,
         )
     ],
+    options=options,
 )

tools/gbench/util.py

@@ -1,5 +1,5 @@
-"""util.py - General utilities for running, loading, and processing benchmarks
+"""util.py - General utilities for running, loading, and processing benchmarks"""
-"""
+
 import json
 import os
 import re
@@ -37,7 +37,7 @@ def is_executable_file(filename):
     elif sys.platform.startswith("win"):
         return magic_bytes == b"MZ"
     else:
-        return magic_bytes == b"\x7FELF"
+        return magic_bytes == b"\x7fELF"
 
 
 def is_json_file(filename):