diff --git a/puppet-openstack_infra_spec_helper.gemspec b/puppet-openstack_infra_spec_helper.gemspec
index a2881c0..061bd84 100644
--- a/puppet-openstack_infra_spec_helper.gemspec
+++ b/puppet-openstack_infra_spec_helper.gemspec
@@ -16,6 +16,10 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  # This is a transitive dep that we define early so that the version
+  # that works for us is found.
+  # rake 12.3.0 requires Ruby version ~> 2.0
+  spec.add_dependency 'rake', ['< 12.3.0']
   # dependencies that are needed to run puppet-lint
   spec.add_dependency 'puppet', [ '~> 3.8']
   spec.add_dependency 'puppetlabs_spec_helper'
diff --git a/run_unit_tests.sh b/run_unit_tests.sh
index 4f76a2e..3387875 100755
--- a/run_unit_tests.sh
+++ b/run_unit_tests.sh
@@ -32,7 +32,10 @@ cd openstack-infra/puppet-openstackci
 
 # Modify Gemfile to use local library and not the one on git
 # so we can actually test the current state of the gem.
-sed -i "s/.*git     => 'https:\/\/git.openstack.org\/openstack-infra\/puppet-openstack_infra_spec_helper.*/      :path => '..\/..',/" Gemfile
+# Note this is largely belts and suspenders for local test runs.
+# puppet-openstackci already attempts to determine if it is running
+# under Zuul and will do the correct thing in that case.
+sed -i "s/:git => 'https:\/\/git.openstack.org\/openstack-infra\/puppet-openstack_infra_spec_helper'}/:path => '..\/..'}/" Gemfile
 
 # Install dependencies
 gem install bundler --no-rdoc --no-ri --verbose