diff --git a/eng/common/pipelines/templates/steps/cache-ps-modules.yml b/eng/common/pipelines/templates/steps/cache-ps-modules.yml
index 563dba40294f..ac136803164f 100644
--- a/eng/common/pipelines/templates/steps/cache-ps-modules.yml
+++ b/eng/common/pipelines/templates/steps/cache-ps-modules.yml
@@ -3,8 +3,12 @@ steps:
     . ./eng/common/scripts/Helpers/PSModule-Helpers.ps1
     Write-Host "##vso[task.setvariable variable=CachedPSModulePath]$global:CurrentUserModulePath"
   displayName: Set PS Modules Cache Directory
+  # Containers should bake modules into the image to save on pipeline time
+  condition: and(succeeded(), eq(variables['Container'], ''))
 - task: Cache@2
   inputs:
     key: 'PSModulePath | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/common/scripts/Import-AzModules.ps1'
     path: $(CachedPSModulePath)
-  displayName: Cache PS Modules
\ No newline at end of file
+  displayName: Cache PS Modules
+  # Containers should bake modules into the image to save on pipeline time
+  condition: and(succeeded(), eq(variables['Container'], ''))