Clang generator: Remember "seen" types when collecting related types

This change includes a small refactoring to remember which types the generator
has already seen during header generations. Otherwise we may loop indefinitely
on certain complex types. One such type is `std::FILE` in Clang's libc++.

PiperOrigin-RevId: 335589238
Change-Id: I5bbe03b6c7fc89c743163f5534075d7912ed4e58

sandboxed_api/tools/clang_generator/generator.cc

@@ -59,9 +59,9 @@ bool GeneratorASTVisitor::VisitFunctionDecl(clang::FunctionDecl* decl) {
        options_.function_names.count(ToStringView(decl->getName())) > 0)) {
     functions_.push_back(decl);
 
-    CollectRelatedTypes(decl->getDeclaredReturnType(), &types_);
+    collector_.CollectRelatedTypes(decl->getDeclaredReturnType());
     for (const clang::ParmVarDecl* param : decl->parameters()) {
-      CollectRelatedTypes(param->getType(), &types_);
+      collector_.CollectRelatedTypes(param->getType());
     }
   }
   return true;
@@ -75,7 +75,7 @@ void GeneratorASTConsumer::HandleTranslationUnit(clang::ASTContext& context) {
                      "AST traversal exited early");
   }
 
-  for (clang::QualType qual : visitor_.types_) {
+  for (clang::QualType qual : visitor_.collector_.collected()) {
     emitter_.CollectType(qual);
   }
   for (clang::FunctionDecl* func : visitor_.functions_) {

sandboxed_api/tools/clang_generator/generator.h

@@ -65,8 +65,9 @@ class GeneratorASTVisitor
  private:
   friend class GeneratorASTConsumer;
 
+  TypeCollector collector_;
+
   std::vector<clang::FunctionDecl*> functions_;
-  QualTypeSet types_;
 
   const GeneratorOptions& options_;
 };

sandboxed_api/tools/clang_generator/types.cc

@@ -30,10 +30,15 @@ bool IsFunctionReferenceType(clang::QualType qual) {
 
 }  // namespace
 
-void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types) {
+void TypeCollector::CollectRelatedTypes(clang::QualType qual) {
+  if (seen_.contains(qual)) {
+    return;
+  }
+  seen_.insert(qual);
+
   if (const auto* typedef_type = qual->getAs<clang::TypedefType>()) {
-    CollectRelatedTypes(typedef_type->getDecl()->getUnderlyingType(), types);
-    types->insert(qual);
+    CollectRelatedTypes(typedef_type->getDecl()->getUnderlyingType());
+    collected_.insert(qual);
     return;
   }
 
@@ -43,26 +48,26 @@ void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types) {
                                         ->getAs<clang::FunctionProtoType>()) {
       // Note: Do not add the function type itself, as this will always be a
       //       pointer argument. We only need to collect all its related types.
-      CollectRelatedTypes(function_type->getReturnType(), types);
+      CollectRelatedTypes(function_type->getReturnType());
       for (const clang::QualType& param : function_type->getParamTypes()) {
-        CollectRelatedTypes(param, types);
+        CollectRelatedTypes(param);
       }
       return;
     }
   }
 
   if (IsPointerOrReference(qual)) {
-    clang::QualType pointee = qual->getPointeeType();
-    while (IsPointerOrReference(pointee)) {
+    clang::QualType pointee = qual;
+    do {
       pointee = pointee->getPointeeType();
-    }
-    CollectRelatedTypes(pointee, types);
+    } while (IsPointerOrReference(pointee));
+    CollectRelatedTypes(pointee);
     return;
   }
 
   // C array with specified constant size (i.e. int a[42])?
   if (const clang::ArrayType* array_type = qual->getAsArrayTypeUnsafe()) {
-    CollectRelatedTypes(array_type->getElementType(), types);
+    CollectRelatedTypes(array_type->getElementType());
     return;
   }
 
@@ -71,19 +76,19 @@ void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types) {
       // Collect the underlying integer type of enum classes as well, as it may
       // be a typedef.
       if (const clang::EnumDecl* decl = enum_type->getDecl(); decl->isFixed()) {
-        CollectRelatedTypes(decl->getIntegerType(), types);
+        CollectRelatedTypes(decl->getIntegerType());
       }
     }
-    types->insert(qual);
+    collected_.insert(qual);
     return;
   }
 
   if (const auto* record_type = qual->getAs<clang::RecordType>()) {
     const clang::RecordDecl* decl = record_type->getDecl();
     for (const clang::FieldDecl* field : decl->fields()) {
-      CollectRelatedTypes(field->getType(), types);
+      CollectRelatedTypes(field->getType());
     }
-    types->insert(qual);
+    collected_.insert(qual);
     return;
   }
 }

sandboxed_api/tools/clang_generator/types.h

@@ -41,20 +41,29 @@ inline bool IsPointerOrReference(clang::QualType qual) {
          qual->isLValueReferenceType() || qual->isRValueReferenceType();
 }
 
-// Computes the transitive closure of all types that a type depends on. Those
-// are types that need to be declared before a declaration of the type denoted
-// by the qual parameter is valid. For example, given
-//   struct SubStruct { bool truth_value; };
-//   struct AggregateStruct {
-//     int int_member;
-//     SubStruct struct_member;
-//   };
-//
-// calling this function on the type "AggregateStruct" yields these types:
-//   int
-//   SubStruct
-//   bool
-void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types);
+class TypeCollector {
+ public:
+  // Computes the transitive closure of all types that a type depends on. Those
+  // are types that need to be declared before a declaration of the type denoted
+  // by the qual parameter is valid. For example, given
+  //   struct SubStruct { bool truth_value; };
+  //   struct AggregateStruct {
+  //     int int_member;
+  //     SubStruct struct_member;
+  //   };
+  //
+  // calling this function on the type "AggregateStruct" yields these types:
+  //   int
+  //   SubStruct
+  //   bool
+  void CollectRelatedTypes(clang::QualType qual);
+
+  QualTypeSet& collected() { return collected_; }
+
+ private:
+  QualTypeSet collected_;
+  QualTypeSet seen_;
+};
 
 // Maps a qualified type to a fully qualified SAPI-compatible type name. This
 // is used for the generated code that invokes the actual function call RPC.