openstack
/
heat-cfntools
Code Issues Proposed changes

Using pipe to handle tgz sources 

For tgz(or tar.bz2) sources, use pipe like `wget -O -
http://www.example.com/a.tar.gz | tar -xvf -` to save disk space
usage.

Change-Id: I59663aed098e8c96d8a41b2d84200f2a1e43a927
Fixes: bug #1192135
This commit is contained in:
JUN JIE NAN 2013-06-26 15:23:15 +08:00
parent 87e08b0b4b
commit 026146ae4e
2 changed files with 94 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
heat_cfntools/cfntools/cfn_helper.py
View File

@ -573,49 +573,60 @@ class SourcesHandler(object):
return '%s/%s' % (tempdir, sp[-1])
def _decompress(self, archive, dest_dir):
cmd_str = ''
LOG.debug("Decompressing")
(r, ext) = os.path.splitext(archive)
if ext == '.tgz':
cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive)
elif ext == '.tbz2':
cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive)
elif ext == '.zip':
cmd_str = 'unzip -d %s %s' % (dest_dir, archive)
elif ext == '.tar':
cmd_str = 'tar -C %s -xf %s' % (dest_dir, archive)
elif ext == '.gz':
(r, ext) = os.path.splitext(r)
if ext:
cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive)
else:
cmd_str = 'gunzip -c %s > %s' % (archive, dest_dir)
elif ext == 'bz2':
(r, ext) = os.path.splitext(r)
if ext:
cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive)
else:
cmd_str = 'bunzip2 -c %s > %s' % (archive, dest_dir)
def _splitext(self, path):
(r, ext) = os.path.splitext(path)
return (r, ext.lower())
def _source_type(self, name):
(r, ext) = self._splitext(name)
if ext == '.gz':
(r, ext2) = self._splitext(r)
if ext2 == '.tar':
ext = '.tgz'
elif ext == '.bz2':
(r, ext2) = self._splitext(r)
if ext2 == '.tar':
ext = '.tbz2'
return ext
def _apply_source_cmd(self, dest, url):
cmd = ""
basename = os.path.basename(url)
stype = self._source_type(basename)
if stype == '.tgz':
cmd = "wget -q -O - '%s' | gunzip | tar -xvf -" % url
elif stype == '.tbz2':
cmd = "wget -q -O - '%s' | bunzip2 | tar -xvf -" % url
elif stype == '.zip':
tmp = self._url_to_tmp_filename(url)
cmd = "wget -q -O '%s' '%s' && unzip -o '%s'" % (tmp, url, tmp)
elif stype == '.tar':
cmd = "wget -q -O - '%s' | tar -xvf -" % url
elif stype == '.gz':
(r, ext) = self._splitext(basename)
cmd = "wget -q -O - '%s' | gunzip > '%s'" % (url, r)
elif stype == '.bz2':
(r, ext) = self._splitext(basename)
cmd = "wget -q -O - '%s' | bunzip2 > '%s'" % (url, r)
else:
pass
return CommandRunner(cmd_str)
if cmd != '':
cmd = "mkdir -p '%s'; cd '%s'; %s" % (dest, dest, cmd)
return cmd
def _apply_source(self, dest, url):
cmd = self._apply_source_cmd(dest, url)
if cmd != '':
runner = CommandRunner(cmd)
runner.run()
def apply_sources(self):
if not self._sources:
return
for dest, url in self._sources.iteritems():
tmp_name = self._url_to_tmp_filename(url)
cmd_str = 'wget -O %s %s' % (tmp_name, url)
try:
os.makedirs(dest)
except OSError as e:
if e.errno == errno.EEXIST:
LOG.debug(str(e))
else:
LOG.exception(e)
decompress_command = self._decompress(tmp_name, dest)
CommandRunner(cmd_str, decompress_command).run()
self._apply_source(dest, url)
class ServicesHandler(object):

50
heat_cfntools/tests/test_cfn_helper.py
View File

@ -685,10 +685,10 @@ class TestSourcesHandler(MockPopenTestCase):
td = os.path.dirname(end_file)
self.m.StubOutWithMock(tempfile, 'mkdtemp')
tempfile.mkdtemp().AndReturn(td)
cmd = ['su', 'root', '-c', 'wget -O %s %s' % (end_file, url)]
er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | gunzip | tar -xvf -"
cmd = ['su', 'root', '-c',
er % (dest, dest, url)]
self.mock_cmd_run(cmd).AndReturn(FakePOpen('Wget good'))
cmd = ['su', 'root', '-c', 'tar -C %s -xzf %s' % (dest, end_file)]
self.mock_cmd_run(cmd).AndReturn(FakePOpen('Tarball good'))
self.m.ReplayAll()
sh = cfn_helper.SourcesHandler(sources)
sh.apply_sources()
@ -706,3 +706,47 @@ class TestSourcesHandler(MockPopenTestCase):
self.addCleanup(os.rmdir, td)
end_file = '%s/file.tar.gz' % td
self._test_apply_sources(url, end_file)
def test_apply_source_cmd(self):
sh = cfn_helper.SourcesHandler({})
er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | %s | tar -xvf -"
dest = '/tmp'
# test tgz
url = 'http://www.example.com/a.tgz'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, url, "gunzip"), cmd)
# test tar.gz
url = 'http://www.example.com/a.tar.gz'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, url, "gunzip"), cmd)
# test tbz2
url = 'http://www.example.com/a.tbz2'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, url, "bunzip2"), cmd)
# test tar.bz2
url = 'http://www.example.com/a.tar.bz2'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, url, "bunzip2"), cmd)
# test zip
er = "mkdir -p '%s'; cd '%s'; wget -q -O '%s' '%s' && unzip -o '%s'"
url = 'http://www.example.com/a.zip'
d = "/tmp/tmp2I0yNK"
tmp = "%s/a.zip" % d
self.m.StubOutWithMock(tempfile, 'mkdtemp')
tempfile.mkdtemp().AndReturn(d)
self.m.ReplayAll()
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, tmp, url, tmp), cmd)
# test gz
er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | %s > '%s'"
url = 'http://www.example.com/a.sh.gz'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, url, "gunzip", "a.sh"), cmd)
# test bz2
url = 'http://www.example.com/a.sh.bz2'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual(er % (dest, dest, url, "bunzip2", "a.sh"), cmd)
# test other
url = 'http://www.example.com/a.sh'
cmd = sh._apply_source_cmd(dest, url)
self.assertEqual("", cmd)