bazel: Add build rules for the new interface generator

This adds a workspace rule that inspects the current system first and
downloads a suitable version of LLVM/Clang from GitHub if it can't
find one. In the latter case, the necessary parts are build from source,
which can take a while (~10-15m, depending on the build machine).

In order to be found, LLVM/Clang system libraries must be version 11
or higher. On Debian/Ubuntu, install `llvm-13-dev` and `libclang-13-dev`.

The new `llvm_config.bzl` implements this logic. It is loosely based on
upstream's https://github.com/llvm/llvm-project/blob/main/utils/bazel/configure.bzl.
Note that due to the way Bazel separates local repositories, we have to
duplictate some of this code.

PiperOrigin-RevId: 438759950
Change-Id: Ia65f473b4cdef6507e3816bf09794ea10963d87a

WORKSPACE

@@ -17,6 +17,11 @@ workspace(name = "com_google_sandboxed_api")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 load("//sandboxed_api/bazel:sapi_deps.bzl", "sapi_deps")
+load(
+    "//sandboxed_api/bazel:llvm_config.bzl",
+    "llvm_configure",
+    "llvm_disable_optional_support_deps",
+)
 
 # Load common dependencies, then Protobuf's
 sapi_deps()
@@ -62,3 +67,14 @@ maybe(
     strip_prefix = "benchmark-3b3de69400164013199ea448f051d94d7fc7d81f",
     urls = ["https://github.com/google/benchmark/archive/3b3de69400164013199ea448f051d94d7fc7d81f.zip"],
 )
+
+# LLVM/libclang
+maybe(
+    llvm_configure,
+    name = "llvm-project",
+    commit = "2c494f094123562275ae688bd9e946ae2a0b4f8b",  # 2022-03-31
+    sha256 = "59b9431ae22f0ea5f2ce880925c0242b32a9e4f1ae8147deb2bb0fc19b53fa0d",
+    system_libraries = True,  # Prefer system libraries
+)
+
+llvm_disable_optional_support_deps()

sandboxed_api/bazel/llvm_config.bzl

@@ -0,0 +1,214 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Repository rule that tries to find system provided LLVM packages."""
+
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+SYSTEM_LLVM_BAZEL = """package(default_visibility = ["//visibility:public"])
+# Create one hidden library with all LLVM headers that depends on all its
+# static library archives. This will be used to provide individual library
+# targets named the same as the upstream Bazel files.
+cc_library(
+    name = "llvm",
+    hdrs = glob([
+        "llvm-project-include/clang-c/**/*.h",
+        "llvm-project-include/clang/**/*.def",
+        "llvm-project-include/clang/**/*.h",
+        "llvm-project-include/clang/**/*.inc",
+        "llvm-project-include/llvm-c/**/*.h",
+        "llvm-project-include/llvm/**/*.def",
+        "llvm-project-include/llvm/**/*.h",
+        "llvm-project-include/llvm/**/*.inc",
+    ]),
+    linkopts = ["-lncurses"],
+    includes = ["llvm-project-include"],
+    srcs = glob([
+        "llvm-project-lib/libLLVM*.a",
+        "llvm-project-lib/libclang*.a",
+    ]),
+    visibility = ["@llvm-project//clang:__pkg__"],
+)
+# Fake support library
+cc_library(name = "Support", deps = ["@llvm-project//llvm:llvm"])
+"""
+
+SYSTEM_CLANG_BAZEL = """package(default_visibility = ["//visibility:public"])
+# Fake libraries that just depend on a big library with all files.
+cc_library(name = "ast", deps = ["@llvm-project//llvm:llvm"])
+cc_library(name = "basic", deps = ["@llvm-project//llvm:llvm"])
+cc_library(name = "driver", deps = ["@llvm-project//llvm:llvm"])
+cc_library(name = "format", deps = ["@llvm-project//llvm:llvm"])
+cc_library(name = "frontend", deps = ["@llvm-project//llvm:llvm"])
+cc_library(name = "lex", deps = ["@llvm-project//llvm:llvm"])
+cc_library(name = "tooling", deps = ["@llvm-project//llvm:llvm"])
+"""
+
+def _use_system_llvm(ctx):
+    found = False
+
+    # Look for LLVM in known places
+    llvm_dirs = ctx.execute(
+        ["ls", "-1"] +
+        [
+            "/usr/lib/llvm-{}/include/llvm/Support/InitLLVM.h".format(ver)
+            for ver in [16, 15, 14, 13, 12, 11]  # Debian
+        ] + [
+            "/usr/include/llvm/Support/InitLLVM.h",  # Fedora and others
+        ],
+    ).stdout.split("\n")[:-1]
+    if llvm_dirs:
+        llvm_dir = llvm_dirs[0].split("/include/llvm/")[0]
+        for suffix in ["llvm", "llvm-c", "clang", "clang-c"]:
+            ctx.symlink(
+                llvm_dir + "/include/" + suffix,
+                "llvm/llvm-project-include/" + suffix,
+            )
+
+        # Try to find the lib directory
+        lib_dirs = ctx.execute(
+            ["ls", "-d", "-1"] +
+            [llvm_dir + "/lib64", llvm_dir + "/lib"],
+        ).stdout.split("\n")[:-1]
+        if lib_dirs:
+            ctx.symlink(lib_dirs[0], "llvm/llvm-project-lib")
+            found = True
+
+    if found:
+        # Create stub targets in sub-packages
+        ctx.file("llvm/BUILD.bazel", SYSTEM_LLVM_BAZEL)
+        ctx.file("clang/BUILD.bazel", SYSTEM_CLANG_BAZEL)
+    return found
+
+def _overlay_directories(ctx, src_path, target_path):
+    bazel_path = src_path.get_child("utils").get_child("bazel")
+    overlay_path = bazel_path.get_child("llvm-project-overlay")
+    script_path = bazel_path.get_child("overlay_directories.py")
+
+    python_bin = ctx.which("python3")
+    if not python_bin:
+        python_bin = ctx.which("python")
+
+    if not python_bin:
+        fail("Failed to find python3 binary")
+
+    cmd = [
+        python_bin,
+        script_path,
+        "--src",
+        src_path,
+        "--overlay",
+        overlay_path,
+        "--target",
+        target_path,
+    ]
+    exec_result = ctx.execute(cmd, timeout = 20)
+
+    if exec_result.return_code != 0:
+        fail(("Failed to execute overlay script: '{cmd}'\n" +
+              "Exited with code {return_code}\n" +
+              "stdout:\n{stdout}\n" +
+              "stderr:\n{stderr}\n").format(
+            cmd = " ".join([str(arg) for arg in cmd]),
+            return_code = exec_result.return_code,
+            stdout = exec_result.stdout,
+            stderr = exec_result.stderr,
+        ))
+
+DEFAULT_LLVM_COMMIT = "2c494f094123562275ae688bd9e946ae2a0b4f8b"  # 2022-03-31
+DEFAULT_LLVM_SHA256 = "59b9431ae22f0ea5f2ce880925c0242b32a9e4f1ae8147deb2bb0fc19b53fa0d"
+
+def _llvm_configure_impl(ctx):
+    commit = ctx.attr.commit
+    sha256 = ctx.attr.sha256
+
+    if ctx.attr.system_libraries:
+        if _use_system_llvm(ctx):
+            return
+        if not commit:
+            fail((
+                "Failed to find LLVM and clang system libraries\n\n" +
+                "Note: You may have to install llvm-13-dev and libclang-13-dev\n" +
+                "      packages (or later versions) first.\n"
+            ))
+
+    if not commit:
+        commit = DEFAULT_LLVM_COMMIT
+        sha256 = DEFAULT_LLVM_SHA256
+
+    ctx.download_and_extract(
+        ["https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = commit)],
+        "llvm-raw",
+        sha256,
+        "",
+        "llvm-project-" + commit,
+    )
+
+    target_path = ctx.path("llvm-raw").dirname
+    src_path = target_path.get_child("llvm-raw")
+    _overlay_directories(ctx, src_path, target_path)
+
+    # Create a starlark file with the requested LLVM targets
+    ctx.file(
+        "llvm/targets.bzl",
+        "llvm_targets = " + str(ctx.attr.targets),
+        executable = False,
+    )
+
+    # Set up C++ toolchain options. LLVM requires at least C++ 14.
+    ctx.file(
+        ".bazelrc",
+        "build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17",
+        executable = False,
+    )
+
+DEFAULT_TARGETS = ["AArch64", "ARM", "PowerPC", "X86"]
+
+llvm_configure = repository_rule(
+    implementation = _llvm_configure_impl,
+    local = True,
+    configure = True,
+    attrs = {
+        "system_libraries": attr.bool(default = True),
+        "commit": attr.string(),
+        "sha256": attr.string(),
+        "targets": attr.string_list(default = DEFAULT_TARGETS),
+    },
+)
+
+def _llvm_zlib_disable_impl(ctx):
+    ctx.file(
+        "BUILD.bazel",
+        """cc_library(name = "zlib", visibility = ["//visibility:public"])""",
+        executable = False,
+    )
+
+llvm_zlib_disable = repository_rule(
+    implementation = _llvm_zlib_disable_impl,
+)
+
+def _llvm_terminfo_disable(ctx):
+    ctx.file(
+        "BUILD.bazel",
+        """cc_library(name = "terminfo", visibility = ["//visibility:public"])""",
+        executable = False,
+    )
+
+llvm_terminfo_disable = repository_rule(
+    implementation = _llvm_terminfo_disable,
+)
+
+def llvm_disable_optional_support_deps():
+    maybe(llvm_zlib_disable, name = "llvm_zlib")
+    maybe(llvm_terminfo_disable, name = "llvm_terminfo")

sandboxed_api/tools/clang_generator/BUILD

@@ -0,0 +1,107 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//sandboxed_api/bazel:build_defs.bzl", "sapi_platform_copts")
+
+licenses(["notice"])
+
+cc_library(
+    name = "generator",
+    srcs = [
+        "diagnostics.cc",
+        "emitter.cc",
+        "generator.cc",
+        "types.cc",
+    ],
+    hdrs = [
+        "diagnostics.h",
+        "emitter.h",
+        "generator.h",
+        "types.h",
+    ],
+    copts = sapi_platform_copts(),
+    deps = [
+        "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/random",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:cord",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:optional",
+        "@llvm-project//clang:ast",
+        "@llvm-project//clang:basic",
+        "@llvm-project//clang:format",
+        "@llvm-project//clang:frontend",
+        "@llvm-project//clang:lex",
+        "@llvm-project//clang:tooling",
+        "@llvm-project//llvm:Support",
+        "//sandboxed_api/util:fileops",
+        "//sandboxed_api/util:status",
+    ],
+)
+
+cc_test(
+    name = "generator_test",
+    srcs = [
+        "emitter_test.cc",
+        "frontend_action_test_util.cc",
+        "frontend_action_test_util.h",
+    ],
+    copts = sapi_platform_copts(),
+    deps = [
+        ":generator",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+        "@llvm-project//clang:basic",
+        "@llvm-project//clang:frontend",
+        "@llvm-project//clang:tooling",
+        "@llvm-project//llvm:Support",
+        "//sandboxed_api:testing",
+        "//sandboxed_api/util:status_matchers",
+    ],
+)
+
+# Clang tool that generates Sandboxed API headers
+cc_binary(
+    name = "generator_tool",
+    srcs = [
+        "compilation_database.cc",
+        "compilation_database.h",
+        "generator_tool.cc",
+    ],
+    copts = sapi_platform_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":generator",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+        "@llvm-project//clang:ast",
+        "@llvm-project//clang:driver",
+        "@llvm-project//clang:frontend",
+        "@llvm-project//clang:tooling",
+        "@llvm-project//llvm:Support",
+        "//sandboxed_api/util:file_helpers",
+        "//sandboxed_api/util:fileops",
+        "//sandboxed_api/util:status",
+    ],
+)