clang_generator: Enable mixed header processing

This implements a custom compilation database to conditionally add the correct
language flags to the compiler frontend. Otherwise, a C header might receive
`--std=c++17` and fail.

Note: All headers are always processed in C++ mode. We expect that headers of
well-behaved C libraries contain `#ifdef __cplusplus`/`extern "C" {}` guards.
PiperOrigin-RevId: 435302048
Change-Id: Ib84e6e1f301ba434999846a012b3f8c16884648e

sandboxed_api/tools/clang_generator/CMakeLists.txt

@@ -56,6 +56,8 @@ target_link_libraries(sapi_generator PUBLIC
 )
 
 add_executable(sapi_generator_tool
+  compilation_database.cc
+  compilation_database.h
   generator_tool.cc
 )
 target_link_libraries(sapi_generator_tool PRIVATE

sandboxed_api/tools/clang_generator/compilation_database.cc

@@ -0,0 +1,226 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "sandboxed_api/tools/clang_generator/compilation_database.h"
+
+#include <algorithm>
+#include <memory>
+
+#include "absl/strings/match.h"
+#include "absl/strings/strip.h"
+#include "clang/Driver/Types.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/Support/Path.h"
+
+namespace sapi {
+
+class WrappingCompilationDatabase : public clang::tooling::CompilationDatabase {
+ public:
+  explicit WrappingCompilationDatabase(
+      clang::tooling::CompilationDatabase& inner)
+      : inner_(&inner) {}
+
+ private:
+  std::vector<clang::tooling::CompileCommand> getCompileCommands(
+      llvm::StringRef file_path) const override {
+    return inner_->getCompileCommands(file_path);
+  }
+
+  std::vector<std::string> getAllFiles() const override {
+    return inner_->getAllFiles();
+  }
+
+  std::vector<clang::tooling::CompileCommand> getAllCompileCommands()
+      const override {
+    return inner_->getAllCompileCommands();
+  }
+
+  clang::tooling::CompilationDatabase* inner_;
+};
+
+std::unique_ptr<clang::tooling::CompilationDatabase> NonOwningCompileCommands(
+    clang::tooling::CompilationDatabase& inner) {
+  return std::make_unique<WrappingCompilationDatabase>(inner);
+}
+
+// Returns the command-line argument for setting the highest C language standard
+// version for a given C++ standard version. If the specified string does not
+// indicate a C++ standard, it is returned unchanged.
+std::string CxxStdToCStd(const std::string& arg) {
+  absl::string_view std = arg;
+  if (!absl::ConsumePrefix(&std, "--std=c++") &&
+      !absl::ConsumePrefix(&std, "-std=c++")) {
+    return arg;
+  }
+  if (std == "23" || std == "2z" || std == "20" || std == "2a") {
+    return "--std=c17";
+  }
+  if (std == "17" || std == "1z" || std == "14" || std == "1y") {
+    return "--std=c11";
+  }
+  if (std == "11" || std == "0x") {
+    return "--std=c99";
+  }
+  return "--std=c89";
+}
+
+class FromCxxAjustedCompilationDatabase
+    : public clang::tooling::CompilationDatabase {
+ public:
+  explicit FromCxxAjustedCompilationDatabase(
+      std::unique_ptr<clang::tooling::CompilationDatabase> inner)
+      : inner_(std::move(inner)) {}
+
+  std::vector<clang::tooling::CompileCommand> getCompileCommands(
+      llvm::StringRef file_path) const override {
+    clang::driver::types::ID id =
+        llvm::sys::path::has_extension(file_path)
+            ? clang::driver::types::lookupTypeForExtension(
+                  llvm::sys::path::extension(file_path).drop_front())
+            : clang::driver::types::TY_CXXHeader;
+
+    std::vector<clang::tooling::CompileCommand> cmds =
+        inner_->getCompileCommands(file_path);
+    for (auto& cmd : cmds) {
+      auto& argv = cmd.CommandLine;
+      if (clang::driver::types::isCXX(id) ||
+          id == clang::driver::types::TY_CHeader) {
+        argv[0] = "clang++";
+        if (id == clang::driver::types::TY_CHeader) {
+          // Parse all headers as C++. Well-behaved headers should have an
+          // include guard.
+          argv.insert(argv.begin() + 1, {"-x", "c++"});
+        }
+      } else {
+        argv[0] = "clang";
+        std::transform(argv.begin(), argv.end(), argv.begin(), CxxStdToCStd);
+      }
+    }
+    return cmds;
+  }
+
+  std::vector<std::string> getAllFiles() const override {
+    return inner_->getAllFiles();
+  }
+
+  std::vector<clang::tooling::CompileCommand> getAllCompileCommands()
+      const override {
+    return {};
+  }
+
+  std::unique_ptr<clang::tooling::CompilationDatabase> inner_;
+};
+
+std::unique_ptr<clang::tooling::CompilationDatabase>
+FromCxxAjustedCompileCommands(
+    std::unique_ptr<clang::tooling::CompilationDatabase> inner) {
+  return std::make_unique<FromCxxAjustedCompilationDatabase>(std::move(inner));
+}
+
+llvm::Expected<OptionsParser> OptionsParser::create(
+    int& argc, const char** argv, llvm::cl::OptionCategory& category,
+    llvm::cl::NumOccurrencesFlag occurrences_flag, const char* overview) {
+  OptionsParser parser;
+  if (llvm::Error err =
+          parser.init(argc, argv, category, occurrences_flag, overview);
+      err) {
+    return err;
+  }
+  return parser;
+}
+
+llvm::Error OptionsParser::init(int& argc, const char** argv,
+                                llvm::cl::OptionCategory& category,
+                                llvm::cl::NumOccurrencesFlag occurrences_flag,
+                                const char* overview) {
+  static auto* build_path = new llvm::cl::opt<std::string>(
+      "p", llvm::cl::desc("Build path"), llvm::cl::Optional,
+      llvm::cl::cat(category), llvm::cl::sub(*llvm::cl::AllSubCommands));
+
+  static auto* source_paths = new llvm::cl::list<std::string>(
+      llvm::cl::Positional, llvm::cl::desc("<source0> [... <sourceN>]"),
+      occurrences_flag, llvm::cl::cat(category),
+      llvm::cl::sub(*llvm::cl::AllSubCommands));
+
+  static auto* args_after = new llvm::cl::list<std::string>(
+      "extra-arg",
+      llvm::cl::desc(
+          "Additional argument to append to the compiler command line"),
+      llvm::cl::cat(category), llvm::cl::sub(*llvm::cl::AllSubCommands));
+
+  static auto* args_before = new llvm::cl::list<std::string>(
+      "extra-arg-before",
+      llvm::cl::desc(
+          "Additional argument to prepend to the compiler command line"),
+      llvm::cl::cat(category), llvm::cl::sub(*llvm::cl::AllSubCommands));
+
+  llvm::cl::ResetAllOptionOccurrences();
+
+  llvm::cl::HideUnrelatedOptions(category);
+
+  {
+    std::string error_message;
+    compilations_ =
+        clang::tooling::FixedCompilationDatabase::loadFromCommandLine(
+            argc, argv, error_message);
+    if (!error_message.empty()) {
+      error_message.append("\n");
+    }
+
+    // Stop initializing if command-line option parsing failed.
+    if (llvm::raw_string_ostream os(error_message);
+        !llvm::cl::ParseCommandLineOptions(argc, argv, overview, &os)) {
+      os.flush();
+      return llvm::make_error<llvm::StringError>(
+          error_message, llvm::inconvertibleErrorCode());
+    }
+  }
+  llvm::cl::PrintOptionValues();
+
+  source_path_list_ = *source_paths;
+  if ((occurrences_flag == llvm::cl::ZeroOrMore ||
+       occurrences_flag == llvm::cl::Optional) &&
+      source_path_list_.empty()) {
+    return llvm::Error::success();
+  }
+  if (!compilations_) {
+    std::string error_message;
+    if (!build_path->empty()) {
+      compilations_ =
+          clang::tooling::CompilationDatabase::autoDetectFromDirectory(
+              *build_path, error_message);
+    } else {
+      compilations_ = clang::tooling::CompilationDatabase::autoDetectFromSource(
+          (*source_paths)[0], error_message);
+    }
+    if (!compilations_) {
+      compilations_.reset(new clang::tooling::FixedCompilationDatabase(
+          ".", std::vector<std::string>()));
+    }
+  }
+  auto adjusting_compilations =
+      std::make_unique<clang::tooling::ArgumentsAdjustingCompilations>(
+          std::move(compilations_));
+  adjuster_ = getInsertArgumentAdjuster(
+      *args_before, clang::tooling::ArgumentInsertPosition::BEGIN);
+  adjuster_ = clang::tooling::combineAdjusters(
+      std::move(adjuster_),
+      getInsertArgumentAdjuster(*args_after,
+                                clang::tooling::ArgumentInsertPosition::END));
+  adjusting_compilations->appendArgumentsAdjuster(adjuster_);
+  compilations_ = std::move(adjusting_compilations);
+  return llvm::Error::success();
+}
+
+}  // namespace sapi

sandboxed_api/tools/clang_generator/compilation_database.h

@@ -0,0 +1,75 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SANDBOXED_API_TOOLS_CLANG_GENERATOR_COMPILATION_DATABASE_H_
+#define SANDBOXED_API_TOOLS_CLANG_GENERATOR_COMPILATION_DATABASE_H_
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "clang/Tooling/ArgumentsAdjusters.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
+
+namespace sapi {
+
+// Returns a CompilationDatabase that redirects to the specified inner database.
+std::unique_ptr<clang::tooling::CompilationDatabase> NonOwningCompileCommands(
+    clang::tooling::CompilationDatabase& inner);
+
+std::unique_ptr<clang::tooling::CompilationDatabase>
+FromCxxAjustedCompileCommands(
+    std::unique_ptr<clang::tooling::CompilationDatabase> inner);
+
+// A parser for options common to all command-line Clang tools. This class
+// behaves the same as clang::tooling::CommonOptionsParser, except that it won't
+// print an error if a compilation database could not be found.
+class OptionsParser {
+ public:
+  static llvm::Expected<OptionsParser> create(
+      int& argc, const char** argv, llvm::cl::OptionCategory& category,
+      llvm::cl::NumOccurrencesFlag occurrences_flag = llvm::cl::OneOrMore,
+      const char* overview = nullptr);
+
+  clang::tooling::CompilationDatabase& getCompilations() {
+    return *compilations_;
+  }
+
+  const std::vector<std::string>& getSourcePathList() const {
+    return source_path_list_;
+  }
+
+  clang::tooling::ArgumentsAdjuster getArgumentsAdjuster() { return adjuster_; }
+
+ private:
+  OptionsParser() = default;
+
+  llvm::Error init(int& argc, const char** argv,
+                   llvm::cl::OptionCategory& category,
+                   llvm::cl::NumOccurrencesFlag occurrences_flag,
+                   const char* overview);
+
+  std::unique_ptr<clang::tooling::CompilationDatabase> compilations_;
+  std::vector<std::string> source_path_list_;
+  clang::tooling::ArgumentsAdjuster adjuster_;
+};
+
+}  // namespace sapi
+
+#endif  // SANDBOXED_API_TOOLS_CLANG_GENERATOR_COMPILATION_DATABASE_H_

sandboxed_api/tools/clang_generator/generator_tool.cc

@@ -13,18 +13,22 @@
 // limitations under the License.
 
 #include <cstdlib>
+#include <memory>
 #include <string>
 #include <vector>
 
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
+#include "absl/strings/match.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_split.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/CommandLine.h"
+#include "sandboxed_api/tools/clang_generator/compilation_database.h"
 #include "sandboxed_api/tools/clang_generator/generator.h"
 #include "sandboxed_api/util/file_helpers.h"
 #include "sandboxed_api/util/fileops.h"
@@ -99,13 +103,14 @@ GeneratorOptions GeneratorOptionsFromFlags(
 }
 
 absl::Status GeneratorMain(int argc, const char** argv) {
-  auto expected_opt_parser = clang::tooling::CommonOptionsParser::create(
+  auto expected_opt_parser = OptionsParser::create(
       argc, argv, *sapi::g_tool_category, llvm::cl::ZeroOrMore,
       "Generates a Sandboxed API header for C/C++ translation units.");
   if (!expected_opt_parser) {
     return absl::InternalError(llvm::toString(expected_opt_parser.takeError()));
   }
-  clang::tooling::CommonOptionsParser& opt_parser = expected_opt_parser.get();
+  OptionsParser& opt_parser = expected_opt_parser.get();
+
   std::vector<std::string> sources = opt_parser.getSourcePathList();
   for (const auto& sapi_in : *sapi::g_sapi_in) {
     sources.push_back(sapi_in);
@@ -117,7 +122,13 @@ absl::Status GeneratorMain(int argc, const char** argv) {
   auto options = sapi::GeneratorOptionsFromFlags(sources);
   sapi::Emitter emitter;
 
-  clang::tooling::ClangTool tool(opt_parser.getCompilations(), sources);
+  std::unique_ptr<clang::tooling::CompilationDatabase> db =
+      FromCxxAjustedCompileCommands(
+          NonOwningCompileCommands(opt_parser.getCompilations()));
+  clang::tooling::ClangTool tool(*db, sources);
+
+  // TODO(cblichmann): Rmove the .isystem files. CMake does not need them, and
+  //                   we can get the information in Bazel from the toolchain.
   if (!sapi::g_sapi_isystem->empty()) {
     std::string isystem_lines;
     SAPI_RETURN_IF_ERROR(sapi::file::GetContents(
@@ -130,6 +141,7 @@ absl::Status GeneratorMain(int argc, const char** argv) {
     tool.appendArgumentsAdjuster(clang::tooling::getInsertArgumentAdjuster(
         isystem, clang::tooling::ArgumentInsertPosition::BEGIN));
   }
+
   if (int result = tool.run(
           absl::make_unique<sapi::GeneratorFactory>(emitter, options).get());
       result != 0) {
@@ -147,7 +159,7 @@ absl::Status GeneratorMain(int argc, const char** argv) {
 
 int main(int argc, const char** argv) {
   if (absl::Status status = sapi::GeneratorMain(argc, argv); !status.ok()) {
-    absl::FPrintF(stderr, "error: %s\n", status.message());
+    absl::FPrintF(stderr, "%s\n", status.message());
     return EXIT_FAILURE;
   }
   return EXIT_SUCCESS;