Tighten diagnostics for calling conventions on variadic functions

Follow-up from r192240.

This makes it an error to use callee-cleanup conventions on variadic
functions, except for __fastcall and __stdcall, which we ignore with
a warning for GCC and MSVC compatibility.

Differential Revision: http://llvm-reviews.chandlerc.com/D1870

llvm-svn: 192308

clang/include/clang/Basic/Specifiers.h

@@ -220,6 +220,19 @@ namespace clang {
     CC_IntelOclBicc // __attribute__((intel_ocl_bicc))
   };
 
+  /// \brief Checks whether the given calling convention is callee-cleanup.
+  inline bool isCalleeCleanup(CallingConv CC) {
+    switch (CC) {
+    case CC_X86StdCall:
+    case CC_X86FastCall:
+    case CC_X86ThisCall:
+    case CC_X86Pascal:
+      return true;
+    default:
+      return false;
+    }
+  }
+
   /// \brief The storage duration for an object (per C++ [basic.stc]).
   enum StorageDuration {
     SD_FullExpression, ///< Full-expression storage duration (for temporaries).

clang/lib/Sema/SemaType.cpp

@@ -4542,22 +4542,26 @@ static bool handleFunctionTypeAttr(TypeProcessingState &state,
     }
   }
 
-  // Diagnose the use of X86 fastcall on varargs or unprototyped functions.
-  if (CC == CC_X86FastCall) {
-    if (isa<FunctionNoProtoType>(fn)) {
-      S.Diag(attr.getLoc(), diag::err_cconv_knr)
-        << FunctionType::getNameForCallConv(CC);
+  // Diagnose use of callee-cleanup calling convention on variadic functions.
+  if (isCalleeCleanup(CC)) {
+    const FunctionProtoType *FnP = dyn_cast<FunctionProtoType>(fn);
+    if (FnP && FnP->isVariadic()) {
+      unsigned DiagID = diag::err_cconv_varargs;
+      // stdcall and fastcall are ignored with a warning for GCC and MS
+      // compatibility.
+      if (CC == CC_X86StdCall || CC == CC_X86FastCall)
+        DiagID = diag::warn_cconv_varargs;
+
+      S.Diag(attr.getLoc(), DiagID) << FunctionType::getNameForCallConv(CC);
       attr.setInvalid();
       return true;
     }
+  }
 
-    const FunctionProtoType *FnP = cast<FunctionProtoType>(fn);
-    if (FnP->isVariadic()) {
-      // In MS compatibility mode, this is just a warning.
-      const LangOptions &L = S.getLangOpts();
-      unsigned DiagID = L.MicrosoftMode ? diag::warn_cconv_varargs
-                                        : diag::err_cconv_varargs;
-      S.Diag(attr.getLoc(), DiagID)
+  // Diagnose the use of X86 fastcall on unprototyped functions.
+  if (CC == CC_X86FastCall) {
+    if (isa<FunctionNoProtoType>(fn)) {
+      S.Diag(attr.getLoc(), diag::err_cconv_knr)
         << FunctionType::getNameForCallConv(CC);
       attr.setInvalid();
       return true;

clang/test/CodeGen/microsoft-call-conv.c

@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm < %s | FileCheck %s
-// RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm -fms-compatibility -DWIN < %s | FileCheck --check-prefix=WIN %s
+// RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm -mrtd < %s | FileCheck %s
+// RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm -fms-compatibility < %s
 
 void __fastcall f1(void);
 void __stdcall f2(void);
@@ -51,9 +52,9 @@ void f8(void) {
 }
 
 // PR12535
-#ifdef WIN
 void __fastcall f9(int x, int y) {};
 // WIN: define x86_fastcallcc void @f9({{.*}})
 void __fastcall f10(int x, ...) {};
 // WIN: define void @f10({{.*}})
-#endif
+void __stdcall f11(int x, ...) {};
+// WIN: define void @f11({{.*}})

clang/test/Sema/callingconv.c

@@ -16,13 +16,12 @@ void __attribute__((fastcall)) test0() { // expected-error {{function with no pr
 void __attribute__((fastcall)) test1(void) {
 }
 
-#ifdef WIN
 void __attribute__((fastcall)) test2(int a, ...) { // expected-warning {{fastcall calling convention ignored on variadic function}}
 }
-#else
-void __attribute__((fastcall)) test2(int a, ...) { // expected-error {{variadic function cannot use fastcall calling convention}}
+void __attribute__((stdcall)) test3(int a, ...) { // expected-warning {{stdcall calling convention ignored on variadic function}}
+}
+void __attribute__((thiscall)) test4(int a, ...) { // expected-error {{variadic function cannot use thiscall calling convention}}
 }
-#endif
 
 void __attribute__((cdecl)) ctest0() {}
 

clang/test/Sema/mrtd.c

@@ -12,8 +12,7 @@ void __attribute__((stdcall)) nonvariadic1(int a, int b, int c);
 void nonvariadic2(int a, int b, int c);
 void __attribute__((stdcall)) nonvariadic2(int a, int b, int c) { }
 
-// expected-note@+2 {{previous declaration is here}}
-// expected-error@+2 {{function declared 'stdcall' here was previously declared without calling convention}}
+// expected-warning@+2 {{stdcall calling convention ignored on variadic function}}
 void variadic(int a, ...);
 void __attribute__((stdcall)) variadic(int a, ...);
 
@@ -34,7 +33,6 @@ __attribute__((cdecl)) extern void (*b)(int, ...);
 extern void (*c)(int, int);
 __attribute__((stdcall)) extern void (*c)(int, int);
 
-// expected-note@+2 {{previous definition is here}}
-// expected-error@+2 {{redefinition of 'd' with a different type: 'void ((*))(int, ...) __attribute__((stdcall))' vs 'void (*)(int, ...)'}}
+// expected-warning@+2 {{stdcall calling convention ignored on variadic function}}
 extern void (*d)(int, ...);
 __attribute__((stdcall)) extern void (*d)(int, ...);

clang/test/SemaCXX/calling-conv-compat.cpp

@@ -248,7 +248,7 @@ namespace Variadic {
 struct A {
   void            member_default(int, ...);
   void __cdecl    member_cdecl(int, ...);
-  void __thiscall member_thiscall(int, ...);
+  void __thiscall member_thiscall(int, ...); // expected-error {{variadic function cannot use thiscall calling convention}}
 };
 
 struct B : public A {