diff --git a/rasa_nlu/evaluate.py b/rasa_nlu/evaluate.py
index 76ae42c7c43a..c61b55702d75 100644
--- a/rasa_nlu/evaluate.py
+++ b/rasa_nlu/evaluate.py
@@ -486,7 +486,7 @@ def determine_token_labels(token, entities, extractors):
 def do_extractors_support_overlap(extractors):
     """Checks if extractors support overlapping entities
     """
-    return extractors is not None and not (
+    return extractors is None or not (
             CRFEntityExtractor.name in extractors)