diff --git a/lib/media/segment_prefetch.js b/lib/media/segment_prefetch.js
index 384844c07e..c78addbbed 100644
--- a/lib/media/segment_prefetch.js
+++ b/lib/media/segment_prefetch.js
@@ -111,6 +111,9 @@ shaka.media.SegmentPrefetch = class {
           shaka.media.SegmentReference.Status.MISSING) {
         prefetchAllowed = false;
       }
+      if (reference.getSegmentData()) {
+        prefetchAllowed = false;
+      }
       if (prefetchAllowed && reference.initSegmentReference) {
         this.prefetchInitSegment(reference.initSegmentReference);
       }
@@ -143,6 +146,10 @@ shaka.media.SegmentPrefetch = class {
       return;
     }
 
+    if (initSegmentReference.getSegmentData()) {
+      return;
+    }
+
     // init segments are ignored from the prefetch limit
     const initSegments = Array.from(this.initSegmentPrefetchMap_.keys());
     const someReference = initSegments.some((reference) => {