Require stdcall etc parameters to be complete on ODR use

Functions using stdcall, fastcall, or vectorcall with C linkage mangle
in the size of the parameter pack. Calculating the size of the pack
requires the parameter types to complete, which may require template
instantiation.

Previously, we would crash during IRgen when requesting the size of
incomplete or uninstantiated types, as in this reduced example:
  struct Foo;
  void __fastcall bar(struct Foo o);
  void (__fastcall *fp)(struct Foo) = &bar;

Reported in Chromium here: https://crbug.com/971245

Differential Revision: https://reviews.llvm.org/D62975

llvm-svn: 363000

clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -2985,6 +2985,9 @@ def err_invalid_attribute_on_virtual_function : Error<
 def warn_declspec_allocator_nonpointer : Warning<
   "ignoring __declspec(allocator) because the function return type %0 is not "
   "a pointer or reference type">, InGroup<IgnoredAttributes>;
+def err_cconv_incomplete_param_type : Error<
+  "parameter %0 must have a complete type to use function %1 with the %2 "
+  "calling convention">;
 
 def ext_cannot_use_trivial_abi : ExtWarn<
   "'trivial_abi' cannot be applied to %0">, InGroup<IgnoredAttributes>;

clang/lib/Sema/SemaExpr.cpp

@@ -14793,6 +14793,81 @@ static bool isPotentiallyConstantEvaluatedContext(Sema &SemaRef) {
   llvm_unreachable("Invalid context");
 }
 
+/// Return true if this function has a calling convention that requires mangling
+/// in the size of the parameter pack.
+static bool funcHasParameterSizeMangling(Sema &S, FunctionDecl *FD) {
+  // These manglings don't do anything on non-Windows or non-x86 platforms, so
+  // we don't need parameter type sizes.
+  const llvm::Triple &TT = S.Context.getTargetInfo().getTriple();
+  if (!TT.isOSWindows() || (TT.getArch() != llvm::Triple::x86 &&
+                            TT.getArch() != llvm::Triple::x86_64))
+    return false;
+
+  // If this is C++ and this isn't an extern "C" function, parameters do not
+  // need to be complete. In this case, C++ mangling will apply, which doesn't
+  // use the size of the parameters.
+  if (S.getLangOpts().CPlusPlus && !FD->isExternC())
+    return false;
+
+  // Stdcall, fastcall, and vectorcall need this special treatment.
+  CallingConv CC = FD->getType()->castAs<FunctionType>()->getCallConv();
+  switch (CC) {
+  case CC_X86StdCall:
+  case CC_X86FastCall:
+  case CC_X86VectorCall:
+    return true;
+  default:
+    break;
+  }
+  return false;
+}
+
+/// Require that all of the parameter types of function be complete. Normally,
+/// parameter types are only required to be complete when a function is called
+/// or defined, but to mangle functions with certain calling conventions, the
+/// mangler needs to know the size of the parameter list. In this situation,
+/// MSVC doesn't emit an error or instantiate templates. Instead, MSVC mangles
+/// the function as _foo@0, i.e. zero bytes of parameters, which will usually
+/// result in a linker error. Clang doesn't implement this behavior, and instead
+/// attempts to error at compile time.
+static void CheckCompleteParameterTypesForMangler(Sema &S, FunctionDecl *FD,
+                                                  SourceLocation Loc) {
+  class ParamIncompleteTypeDiagnoser : public Sema::TypeDiagnoser {
+    FunctionDecl *FD;
+    ParmVarDecl *Param;
+
+  public:
+    ParamIncompleteTypeDiagnoser(FunctionDecl *FD, ParmVarDecl *Param)
+        : FD(FD), Param(Param) {}
+
+    void diagnose(Sema &S, SourceLocation Loc, QualType T) override {
+      CallingConv CC = FD->getType()->castAs<FunctionType>()->getCallConv();
+      StringRef CCName;
+      switch (CC) {
+      case CC_X86StdCall:
+        CCName = "stdcall";
+        break;
+      case CC_X86FastCall:
+        CCName = "fastcall";
+        break;
+      case CC_X86VectorCall:
+        CCName = "vectorcall";
+        break;
+      default:
+        llvm_unreachable("CC does not need mangling");
+      }
+
+      S.Diag(Loc, diag::err_cconv_incomplete_param_type)
+          << Param->getDeclName() << FD->getDeclName() << CCName;
+    }
+  };
+
+  for (ParmVarDecl *Param : FD->parameters()) {
+    ParamIncompleteTypeDiagnoser Diagnoser(FD, Param);
+    S.RequireCompleteType(Loc, Param->getType(), Diagnoser);
+  }
+}
+
 namespace {
 enum class OdrUseContext {
   /// Declarations in this context are not odr-used.
@@ -15038,6 +15113,12 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func,
         UndefinedButUsed.insert(std::make_pair(Func->getCanonicalDecl(), Loc));
     }
 
+    // Some x86 Windows calling conventions mangle the size of the parameter
+    // pack into the name. Computing the size of the parameters requires the
+    // parameter types to be complete. Check that now.
+    if (funcHasParameterSizeMangling(*this, Func))
+      CheckCompleteParameterTypesForMangler(*this, Func, Loc);
+
     Func->markUsed(Context);
 
     if (LangOpts.OpenMP && LangOpts.OpenMPIsDevice)

clang/test/Sema/calling-conv-complete-params.c

@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -fsyntax-only -fms-extensions -verify -triple i686-pc-win32 %s
+// RUN: %clang_cc1 -fsyntax-only -fms-extensions -verify -triple x86_64-pc-win32 %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -fms-extensions -verify -triple i686-pc-win32 %s
+// RUN: %clang_cc1 -x c++ -DEXTERN_C='extern "C"' -fsyntax-only -fms-extensions -verify -triple i686-pc-win32 %s
+
+#ifndef EXTERN_C
+#define EXTERN_C
+#if defined(__cplusplus)
+#define EXPECT_NODIAG
+// expected-no-diagnostics
+#endif
+#endif
+
+#ifndef EXPECT_NODIAG
+// expected-note-re@+2 1+ {{forward declaration of '{{(struct )?}}Foo'}}
+#endif
+struct Foo;
+
+EXTERN_C void __stdcall fwd_std(struct Foo p);
+#if !defined(EXPECT_NODIAG) && defined(_M_IX86)
+// expected-error@+2 {{parameter 'p' must have a complete type to use function 'fwd_std' with the stdcall calling convention}}
+#endif
+void (__stdcall *fp_fwd_std)(struct Foo) = &fwd_std;
+
+EXTERN_C void __fastcall fwd_fast(struct Foo p);
+#if !defined(EXPECT_NODIAG) && defined(_M_IX86)
+// expected-error@+2 {{parameter 'p' must have a complete type to use function 'fwd_fast' with the fastcall calling convention}}
+#endif
+void (__fastcall *fp_fwd_fast)(struct Foo) = &fwd_fast;
+
+EXTERN_C void __vectorcall fwd_vector(struct Foo p);
+#if !defined(EXPECT_NODIAG)
+// expected-error@+2 {{parameter 'p' must have a complete type to use function 'fwd_vector' with the vectorcall calling convention}}
+#endif
+void (__vectorcall *fp_fwd_vector)(struct Foo) = &fwd_vector;
+
+#if defined(__cplusplus)
+template <typename T> struct TemplateWrapper {
+#ifndef EXPECT_NODIAG
+  // expected-error@+2 {{field has incomplete type 'Foo'}}
+#endif
+  T field;
+};
+
+EXTERN_C void __vectorcall tpl_ok(TemplateWrapper<int> p);
+void(__vectorcall *fp_tpl_ok)(TemplateWrapper<int>) = &tpl_ok;
+
+EXTERN_C void __vectorcall tpl_fast(TemplateWrapper<Foo> p);
+#ifndef EXPECT_NODIAG
+// expected-note@+2 {{requested here}}
+#endif
+void(__vectorcall *fp_tpl_fast)(TemplateWrapper<Foo>) = &tpl_fast;
+#endif