diff --git a/CHANGELOG.md b/CHANGELOG.md
index f38db3b..b9f9025 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,9 @@
 # Unreleased
 
 * Use `RbConfig::CONFIG["rubylibdir"]` instead of `RbConfig::CONFIG["libdir"]` to check for stdlib files. See #431.
+* Fix the cached version of `YAML.load_file` being slightly more permissive than the default `Psych` one. See #434.
+  `Date` and `Time` values are now properly rejected, as well as aliases.
+  If this causes a regression in your application, it is recommended to load *trusted* YAML files with `YAML.unsafe_load_file`.
 
 # 1.15.0
 
diff --git a/lib/bootsnap/compile_cache/yaml.rb b/lib/bootsnap/compile_cache/yaml.rb
index 1c90b8e..7a3b871 100644
--- a/lib/bootsnap/compile_cache/yaml.rb
+++ b/lib/bootsnap/compile_cache/yaml.rb
@@ -65,7 +65,7 @@ def init!
           end
 
           unless const_defined?(:NoTagsVisitor)
-            visitor = Class.new(Psych::Visitors::ToRuby) do
+            visitor = Class.new(Psych::Visitors::NoAliasRuby) do
               def visit(target)
                 if target.tag
                   raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
@@ -129,7 +129,10 @@ def strict_load(payload)
           ast = ::YAML.parse(payload)
           return ast unless ast
 
-          NoTagsVisitor.create.visit(ast)
+          loader = ::Psych::ClassLoader::Restricted.new(["Symbol"], [])
+          scanner = ::Psych::ScalarScanner.new(loader)
+
+          NoTagsVisitor.new(scanner, loader).visit(ast)
         end
       end
 
diff --git a/test/compile_cache/yaml_test.rb b/test/compile_cache/yaml_test.rb
index 9aaf78c..5fb9de9 100644
--- a/test/compile_cache/yaml_test.rb
+++ b/test/compile_cache/yaml_test.rb
@@ -37,6 +37,38 @@ def test_yaml_strict_load
     assert_equal expected, document
   end
 
+  def test_strict_load_reject_dates
+    error = assert_raises Psych::DisallowedClass do
+      ::Bootsnap::CompileCache::YAML.strict_load(<<~YAML)
+        ---
+        :foo: 2023-01-20
+      YAML
+    end
+    assert_includes error.message, "Date"
+  end
+
+  def test_strict_load_reject_times
+    error = assert_raises Psych::DisallowedClass do
+      ::Bootsnap::CompileCache::YAML.strict_load(<<~YAML)
+        ---
+        :foo: 2023-01-20 13:18:31.083375000 -05:00
+      YAML
+    end
+    assert_includes error.message, "Time"
+  end
+
+  def test_strict_load_reject_aliases
+    assert_raises Psych::BadAlias do
+      ::Bootsnap::CompileCache::YAML.strict_load(<<~YAML)
+        ---
+        foo: &foo
+          a: b
+        bar:
+          <<: *foo
+      YAML
+    end
+  end
+
   def test_yaml_tags
     error = assert_raises Bootsnap::CompileCache::YAML::UnsupportedTags do
       ::Bootsnap::CompileCache::YAML.strict_load("!many Boolean")