diff --git a/codecov.yml b/codecov.yml
index 99765917937cec..2d12cb5c84b41e 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -5,16 +5,20 @@ coverage:
         threshold: 1%
 codecov:
   notify:
-    after_n_builds: 2
+    # Code coverage is collected by 5 configs: codecov_test[12], onnx[12] and windows_test1
+    after_n_builds: 5
 comment:
   layout: "diff"
   behavior: once
   require_changes: true
   require_base: yes
   require_head: yes
-  after_n_builds: 2
+  after_n_builds: 5
   branches:
     - "master"
+# Disable inline comments that this code is not covered
+github_checks:
+    annotations: false
 fixes:
   - "/opt/conda/lib/python3.8/site-packages/::project/"
   - "C:/Users/circleci/project/build/win_tmp/build/::project/"