diff --git a/tests/test_modeling_megatron_gpt2.py b/tests/test_modeling_megatron_gpt2.py
index a1f7c472e3..cde07f34a8 100644
--- a/tests/test_modeling_megatron_gpt2.py
+++ b/tests/test_modeling_megatron_gpt2.py
@@ -31,6 +31,7 @@ if is_torch_available():
 @require_tokenizers
 class MegatronGPT2IntegrationTest(unittest.TestCase):
     @slow
+    @unittest.skip("Model is not available.")
     def test_inference_no_head(self):
         directory = "nvidia/megatron-gpt2-345m/"
         if "MYDIR" in os.environ: