diff --git a/mmcv/utils/registry.py b/mmcv/utils/registry.py
index ebef139c72..b0a0d57e3e 100644
--- a/mmcv/utils/registry.py
+++ b/mmcv/utils/registry.py
@@ -139,9 +139,12 @@ def infer_scope():
         Returns:
             str: The inferred scope name.
         """
-        # inspect.stack() trace where this function is called, the index-2
-        # indicates the frame where `infer_scope()` is called
-        filename = inspect.getmodule(inspect.stack()[2][0]).__name__
+        # We access the caller using inspect.currentframe() instead of
+        # inspect.stack() for performance reasons. See details in PR #1844
+        frame = inspect.currentframe()
+        # get the frame where `infer_scope()` is called
+        infer_scope_caller = frame.f_back.f_back
+        filename = inspect.getmodule(infer_scope_caller).__name__
         split_filename = filename.split('.')
         return split_filename[0]