Comment parsing: when comment ranges are deserialized from multiple modules,

correctly order comments in SourceManager::isBeforeInTranslationUnit() order

Unfortunately, this is not as simple as it was implemented previously, and
actually requires doing a merge sort.

llvm-svn: 204936

clang/include/clang/AST/RawCommentList.h

@@ -193,9 +193,7 @@ private:
   SourceManager &SourceMgr;
   std::vector<RawComment *> Comments;
 
-  void addCommentsToFront(const std::vector<RawComment *> &C) {
+  void addDeserializedComments(ArrayRef<RawComment *> DeserializedComments);
-    Comments.insert(Comments.begin(), C.begin(), C.end());
-  }
 
   friend class ASTReader;
 };

clang/lib/AST/ASTContext.cpp

@@ -63,6 +63,13 @@ enum FloatingRank {
 RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
   if (!CommentsLoaded && ExternalSource) {
     ExternalSource->ReadComments();
+
+#ifndef NDEBUG
+    ArrayRef<RawComment *> RawComments = Comments.getComments();
+    assert(std::is_sorted(RawComments.begin(), RawComments.end(),
+                          BeforeThanCompare<RawComment>(SourceMgr)));
+#endif
+
     CommentsLoaded = true;
   }
 

clang/lib/AST/RawCommentList.cpp

@@ -251,3 +251,15 @@ void RawCommentList::addComment(const RawComment &RC,
     Comments.push_back(new (Allocator) RawComment(RC));
   }
 }
+
+void RawCommentList::addDeserializedComments(ArrayRef<RawComment *> DeserializedComments) {
+  std::vector<RawComment *> MergedComments;
+  MergedComments.reserve(Comments.size() + DeserializedComments.size());
+
+  std::merge(Comments.begin(), Comments.end(),
+             DeserializedComments.begin(), DeserializedComments.end(),
+             std::back_inserter(MergedComments),
+             BeforeThanCompare<RawComment>(SourceMgr));
+  std::swap(Comments, MergedComments);
+}
+

clang/lib/Serialization/ASTReader.cpp

@@ -7688,6 +7688,7 @@ void ASTReader::ReadComments() {
        I = CommentsCursors.begin(),
        E = CommentsCursors.end();
        I != E; ++I) {
+    Comments.clear();
     BitstreamCursor &Cursor = I->first;
     serialization::ModuleFile &F = *I->second;
     SavedStreamPosition SavedPosition(Cursor);
@@ -7696,7 +7697,7 @@ void ASTReader::ReadComments() {
     while (true) {
       llvm::BitstreamEntry Entry =
         Cursor.advanceSkippingSubblocks(BitstreamCursor::AF_DontPopBlockAtEnd);
-      
+
       switch (Entry.Kind) {
       case llvm::BitstreamEntry::SubBlock: // Handled for us already.
       case llvm::BitstreamEntry::Error:
@@ -7726,9 +7727,9 @@ void ASTReader::ReadComments() {
       }
       }
     }
-  NextCursor:;
+  NextCursor:
+    Context.Comments.addDeserializedComments(Comments);
   }
-  Context.Comments.addCommentsToFront(Comments);
 }
 
 void ASTReader::finishPendingActions() {

clang/test/Index/Inputs/Frameworks/DocCommentsA.framework/Headers/DocCommentsA.h

@@ -0,0 +1,8 @@
+/// Comment for 'functionFromDocCommentsA1'.
+void functionFromDocCommentsA1(void);
+
+#import <DocCommentsC/DocCommentsC.h>
+
+/// Comment for 'functionFromDocCommentsA2'.
+void functionFromDocCommentsA2(void);
+

clang/test/Index/Inputs/Frameworks/DocCommentsB.framework/Headers/DocCommentsB.h

@@ -0,0 +1,7 @@
+/// Comment for 'functionFromDocCommentsB1'.
+void functionFromDocCommentsB1(void);
+
+#import <DocCommentsC/DocCommentsC.h>
+
+/// Comment for 'functionFromDocCommentsB2'.
+void functionFromDocCommentsB2(void);

clang/test/Index/Inputs/Frameworks/DocCommentsC.framework/Headers/DocCommentsC.h

@@ -0,0 +1,2 @@
+/// Comment for 'functionFromDocCommentsC'.
+void functionFromDocCommentsC(void);

clang/test/Index/annotate-comments-objc.m

@@ -3,6 +3,12 @@
 #ifndef HEADER
 #define HEADER
 
+/// Comment for 'functionBeforeImports'.
+void functionBeforeImports(void);
+
+#import <DocCommentsA/DocCommentsA.h>
+#import <DocCommentsB/DocCommentsB.h>
+
 @class NSString;
 
 //===---
@@ -33,13 +39,15 @@
 // RUN: mkdir %t
 
 // Check that we serialize comment source locations properly.
-// RUN: %clang_cc1 -emit-pch -o %t/out.pch %s
+// RUN: %clang_cc1 -emit-pch -o %t/out.pch -F %S/Inputs/Frameworks %s
-// RUN: %clang_cc1 -include-pch %t/out.pch -fsyntax-only %s
+// RUN: %clang_cc1 -include-pch %t/out.pch -F %S/Inputs/Frameworks -fsyntax-only %s
 
-// RUN: c-index-test -test-load-source all -comments-xml-schema=%S/../../bindings/xml/comment-xml-schema.rng %s > %t/out.c-index-direct
+// RUN: c-index-test -test-load-source all -comments-xml-schema=%S/../../bindings/xml/comment-xml-schema.rng %s -F %S/Inputs/Frameworks > %t/out.c-index-direct
-// RUN: c-index-test -test-load-tu %t/out.pch all > %t/out.c-index-pch
+// RUN: c-index-test -test-load-source all -comments-xml-schema=%S/../../bindings/xml/comment-xml-schema.rng %s -F %S/Inputs/Frameworks -fmodules > %t/out.c-index-modules
+// RUN: c-index-test -test-load-tu %t/out.pch all -F %S/Inputs/Frameworks > %t/out.c-index-pch
 
 // RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-direct
+// RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-modules
 // RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-pch
 
 // Declarations without Doxygen comments should not pick up some Doxygen comments.
@@ -60,6 +68,7 @@
 // WRONG-NOT: CommentXMLInvalid
 
 // RUN: FileCheck %s < %t/out.c-index-direct
+// RUN: FileCheck %s < %t/out.c-index-modules
 // RUN: FileCheck %s < %t/out.c-index-pch
 
 // These CHECK lines are not located near the code on purpose.  This test
@@ -67,12 +76,18 @@
 // Adding a non-documentation comment with CHECK line between every two
 // documentation comments will only test a single code path.
 //
-// CHECK: annotate-comments-objc.m:17:50: ObjCPropertyDecl=property1_isdoxy1:{{.*}} property1_isdoxy1 IS_DOXYGEN_SINGLE
+// CHECK-DAG: annotate-comments-objc.m:7:6: FunctionDecl=functionBeforeImports:{{.*}} BriefComment=[Comment for 'functionBeforeImports'.]
-// CHECK: annotate-comments-objc.m:18:50: ObjCPropertyDecl=property1_isdoxy2:{{.*}} property1_isdoxy2 IS_DOXYGEN_SINGLE
+// CHECK-DAG: DocCommentsA.h:2:6: FunctionDecl=functionFromDocCommentsA1:{{.*}} BriefComment=[Comment for 'functionFromDocCommentsA1'.]
-// CHECK: annotate-comments-objc.m:19:50: ObjCPropertyDecl=property1_isdoxy3:{{.*}} property1_isdoxy3 IS_DOXYGEN_SINGLE
+// CHECK-DAG: DocCommentsA.h:7:6: FunctionDecl=functionFromDocCommentsA2:{{.*}} BriefComment=[Comment for 'functionFromDocCommentsA2'.]
-// CHECK: annotate-comments-objc.m:20:50: ObjCPropertyDecl=property1_isdoxy4:{{.*}} property1_isdoxy4 IS_DOXYGEN_SINGLE
+// CHECK-DAG: DocCommentsB.h:2:6: FunctionDecl=functionFromDocCommentsB1:{{.*}} BriefComment=[Comment for 'functionFromDocCommentsB1'.]
-// CHECK: annotate-comments-objc.m:23:9: ObjCInstanceMethodDecl=method1_isdoxy1:{{.*}} method1_isdoxy1 IS_DOXYGEN_SINGLE
+// CHECK-DAG: DocCommentsB.h:7:6: FunctionDecl=functionFromDocCommentsB2:{{.*}} BriefComment=[Comment for 'functionFromDocCommentsB2'.]
-// CHECK: annotate-comments-objc.m:24:9: ObjCInstanceMethodDecl=method1_isdoxy2:{{.*}} method1_isdoxy2 IS_DOXYGEN_SINGLE
+// CHECK-DAG: DocCommentsC.h:2:6: FunctionDecl=functionFromDocCommentsC:{{.*}} BriefComment=[Comment for 'functionFromDocCommentsC'.]
-// CHECK: annotate-comments-objc.m:25:9: ObjCInstanceMethodDecl=method1_isdoxy3:{{.*}} method1_isdoxy3 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:23:50: ObjCPropertyDecl=property1_isdoxy1:{{.*}} property1_isdoxy1 IS_DOXYGEN_SINGLE
-// CHECK: annotate-comments-objc.m:26:9: ObjCInstanceMethodDecl=method1_isdoxy4:{{.*}} method1_isdoxy4 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:24:50: ObjCPropertyDecl=property1_isdoxy2:{{.*}} property1_isdoxy2 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:25:50: ObjCPropertyDecl=property1_isdoxy3:{{.*}} property1_isdoxy3 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:26:50: ObjCPropertyDecl=property1_isdoxy4:{{.*}} property1_isdoxy4 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:29:9: ObjCInstanceMethodDecl=method1_isdoxy1:{{.*}} method1_isdoxy1 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:30:9: ObjCInstanceMethodDecl=method1_isdoxy2:{{.*}} method1_isdoxy2 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:31:9: ObjCInstanceMethodDecl=method1_isdoxy3:{{.*}} method1_isdoxy3 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments-objc.m:32:9: ObjCInstanceMethodDecl=method1_isdoxy4:{{.*}} method1_isdoxy4 IS_DOXYGEN_SINGLE