diff --git a/examples/hans/test_hans.py b/examples/hans/test_hans.py
index 56416b28bd..99b6f24671 100644
--- a/examples/hans/test_hans.py
+++ b/examples/hans/test_hans.py
@@ -255,7 +255,7 @@ def evaluate(args, model, tokenizer, prefix=""):
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
         # multi-gpu eval
-        if args.n_gpu > 1:
+        if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
             model = torch.nn.DataParallel(model)
 
         # Eval!
diff --git a/examples/mm-imdb/run_mmimdb.py b/examples/mm-imdb/run_mmimdb.py
index ab60b40951..f74ea575b7 100644
--- a/examples/mm-imdb/run_mmimdb.py
+++ b/examples/mm-imdb/run_mmimdb.py
@@ -278,7 +278,7 @@ def evaluate(args, model, tokenizer, criterion, prefix=""):
     )
 
     # multi-gpu eval
-    if args.n_gpu > 1:
+    if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
         model = torch.nn.DataParallel(model)
 
     # Eval!
diff --git a/examples/ner/run_ner.py b/examples/ner/run_ner.py
index ba08e51da8..e280037ad8 100644
--- a/examples/ner/run_ner.py
+++ b/examples/ner/run_ner.py
@@ -253,7 +253,7 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=""
     eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
     # multi-gpu evaluate
-    if args.n_gpu > 1:
+    if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
         model = torch.nn.DataParallel(model)
 
     # Eval!
diff --git a/examples/run_language_modeling.py b/examples/run_language_modeling.py
index 92807dd79c..859bc164c3 100644
--- a/examples/run_language_modeling.py
+++ b/examples/run_language_modeling.py
@@ -427,7 +427,7 @@ def evaluate(args, model: PreTrainedModel, tokenizer: PreTrainedTokenizer, prefi
     )
 
     # multi-gpu evaluate
-    if args.n_gpu > 1:
+    if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
         model = torch.nn.DataParallel(model)
 
     # Eval!
diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py
index 578ce01226..341733b8a0 100644
--- a/examples/run_multiple_choice.py
+++ b/examples/run_multiple_choice.py
@@ -256,7 +256,7 @@ def evaluate(args, model, tokenizer, prefix="", test=False):
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
         # multi-gpu evaluate
-        if args.n_gpu > 1:
+        if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
             model = torch.nn.DataParallel(model)
 
         # Eval!
diff --git a/examples/run_xnli.py b/examples/run_xnli.py
index 9b64d17dfd..7c2790f53d 100644
--- a/examples/run_xnli.py
+++ b/examples/run_xnli.py
@@ -266,7 +266,7 @@ def evaluate(args, model, tokenizer, prefix=""):
         eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 
         # multi-gpu eval
-        if args.n_gpu > 1:
+        if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
             model = torch.nn.DataParallel(model)
 
         # Eval!