diff --git a/jdaviz/configs/default/plugins/plot_options/plot_options.py b/jdaviz/configs/default/plugins/plot_options/plot_options.py
index 4ad3f377f3..9cd316df79 100644
--- a/jdaviz/configs/default/plugins/plot_options/plot_options.py
+++ b/jdaviz/configs/default/plugins/plot_options/plot_options.py
@@ -36,6 +36,8 @@
 
 __all__ = ['PlotOptions']
 
+RANDOM_SUBSET_SIZE = 10_000
+
 
 def _register_random_cmap(
     cmap_name,
@@ -894,6 +896,7 @@ def _update_stretch_hist_sync(self, msg={}):
     @skip_if_no_updates_since_last_active()
     @with_spinner('stretch_hist_spinner')
     def _update_stretch_histogram(self, msg={}):
+
         if not hasattr(self, 'viewer'):  # pragma: no cover
             # plugin hasn't been fully initialized yet
             return
@@ -999,15 +1002,21 @@ def _update_stretch_histogram(self, msg={}):
             arr = comp.data
             sub_data = arr.ravel()
 
-        # filter out nans (or else bqplot will fail)
-        if np.any(np.isnan(sub_data)):
-            sub_data = sub_data[~np.isnan(sub_data)]
-
+        self.stretch_histogram.viewer.state.random_subset = RANDOM_SUBSET_SIZE
         self.stretch_histogram._update_data('histogram', x=sub_data)
 
         if len(sub_data) > 0:
-            interval = PercentileInterval(95)
-            hist_lims = interval.get_limits(sub_data)
+
+            # Use glue to compute the statistics since this allows us to use
+            # a random subset of the data to compute the histogram
+            glue_data = self.stretch_histogram.app.data_collection['histogram']
+            hist_lims = (
+                glue_data.compute_statistic('percentile', glue_data.id['x'],
+                                            percentile=2.5, random_subset=RANDOM_SUBSET_SIZE),
+                glue_data.compute_statistic('percentile', glue_data.id['x'],
+                                            percentile=97.5, random_subset=RANDOM_SUBSET_SIZE)
+            )
+
             # set the stepsize for vmin/vmax to be approximately 1% of the range of the
             # histogram (within the percentile interval), rounded to 1-2 significant digits
             # to avoid random step sizes.  This logic is somewhat arbitrary and can be safely