diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 45705f11..65f7029b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -41,6 +41,12 @@ jobs:
       echo -e "install: --no-document --env-shebang\nupdate:  --no-document --env-shebang" > ~/.gemrc
       bundle install
     displayName: "bundle install"
+    
+  - script: cd ./CalSmokeApp && bundle install
+    displayName: "CalSmokeApp bundle install"
+
+  - script: cd ./CalSmokeApp && make app-cal
+    displayName: "make app-cal"
 
   - script: bundle exec bin/ci/cucumber.rb
     displayName: "cucumber tests"
\ No newline at end of file
diff --git a/bin/ci/cucumber.rb b/bin/ci/cucumber.rb
index 71418d9d..790c895e 100755
--- a/bin/ci/cucumber.rb
+++ b/bin/ci/cucumber.rb
@@ -24,24 +24,6 @@ def select_sim_by_name(simctl, regex)
     FileUtils.rm_rf("reports")
     FileUtils.mkdir_p("reports")
 
-    if Luffa::Environment.jenkins_ci?
-      Luffa::Gem.update_rubygems
-      Luffa::Gem.uninstall_gem("calabash-cucumber")
-      Luffa::Gem.uninstall_gem("run_loop")
-    end
-
-    Luffa.unix_command('bundle update',
-                       {:pass_msg => 'bundled',
-                        :fail_msg => 'could not bundle'})
-
-    Luffa.unix_command('make clean',
-                       {:pass_msg => 'cleaned',
-                        :fail_msg => 'could not clean'})
-
-    Luffa.unix_command('make app-cal',
-                       {:pass_msg => 'built app',
-                        :fail_msg => 'could not build app'})
-
     xcode = RunLoop::Xcode.new
     simctl = RunLoop::Simctl.new