openstack
/
glance
Code Issues Proposed changes

Remove sendfile support 

pysendfile[1] was added as an optional dependency but the library
hasn't been maintained and has got no release since 2014.

What is worse, the sendfile implementation is not actually working
since The SendFileIterator class was removed[2]. (Follow-up[3] removed
the remaining reference to the class). The broken implementation has
not been detected because the client is not currently used to upload
contents.

Remove the incomplete implementation to get rid of the dependency on
the unmaintained library.

[1] https://pypi.org/project/pysendfile/
[2] 76c3620c7e
[3] 0c151d7d7e

Closes-Bug: #2062573
Change-Id: Ia4784f59d16660e8d40c0e409f092ac4e46870b4
This commit is contained in:
Takashi Kajinami 2024-04-19 21:26:24 +09:00
parent 0bcd6cd71c
commit 205ca3336a
4 changed files with 4 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/source/admin/requirements.rst
View File

@ -23,9 +23,8 @@ External Requirements Affecting Glance
Like other OpenStack projects, Glance uses some external libraries for a subset
of its features. Some examples include the ``qemu-img`` utility used by the
tasks feature, ``sendfile`` to utilize the "zero-copy" way of copying data
faster, ``pydev`` to debug using popular IDEs, ``python-xattr`` for Image Cache
using "xattr" driver.
tasks feature, ``pydev`` to debug using popular IDEs, ``python-xattr`` for
Image Cache using "xattr" driver.
On the other hand, if ``dnspython`` is installed in the environment, Glance
provides a workaround to make it work with IPV6.

40
glance/common/client.py
View File

@ -19,7 +19,6 @@
import collections.abc
import copy
import errno
import functools
import http.client
import os
@ -35,12 +34,6 @@ except ImportError:
import osprofiler.web
try:
import sendfile # noqa
SENDFILE_SUPPORTED = True
except ImportError:
SENDFILE_SUPPORTED = False
from oslo_log import log as logging
from oslo_utils import encodeutils
from oslo_utils import netutils
@ -446,12 +439,6 @@ class BaseClient(object):
def _filelike(body):
return hasattr(body, 'read')
def _sendbody(connection, iter):
connection.endheaders()
for sent in iter:
# iterator has done the heavy lifting
pass
def _chunkbody(connection, iter):
connection.putheader('Transfer-Encoding', 'chunked')
connection.endheaders()
@ -469,22 +456,15 @@ class BaseClient(object):
elif _filelike(body) or self._iterable(body):
c.putrequest(method, path)
use_sendfile = self._sendable(body)
# According to HTTP/1.1, Content-Length and Transfer-Encoding
# conflict.
for header, value in headers.items():
if use_sendfile or header.lower() != 'content-length':
if header.lower() != 'content-length':
c.putheader(header, str(value))
iter = utils.chunkreadable(body)
if use_sendfile:
# send actual file without copying into userspace
_sendbody(c, iter)
else:
# otherwise iterate and chunk
_chunkbody(c, iter)
_chunkbody(c, iter)
else:
raise TypeError('Unsupported image type: %s' % body.__class__)
@ -528,22 +508,6 @@ class BaseClient(object):
except (socket.error, IOError) as e:
raise exception.ClientConnectionError(e)
def _seekable(self, body):
# pipes are not seekable, avoids sendfile() failure on e.g.
# cat /path/to/image | glance add ...
# or where add command is launched via popen
try:
os.lseek(body.fileno(), 0, os.SEEK_CUR)
return True
except OSError as e:
return (e.errno != errno.ESPIPE)
def _sendable(self, body):
return (SENDFILE_SUPPORTED and
hasattr(body, 'fileno') and
self._seekable(body) and
not self.use_ssl)
def _iterable(self, body):
return isinstance(body, collections.abc.Iterable)

40
glance/tests/stubs.py
View File

@ -15,23 +15,11 @@
"""Stubouts, mocks and fixtures for the test suite"""
import os
try:
import sendfile
SENDFILE_SUPPORTED = True
except ImportError:
SENDFILE_SUPPORTED = False
import routes
import webob
from glance.api.middleware import context
from glance.api.v2 import router
import glance.common.client
DEBUG = False
def stub_out_store_server(stubs, base_dir, **kwargs):
@ -49,23 +37,10 @@ def stub_out_store_server(stubs, base_dir, **kwargs):
def fileno(self):
return 42
class FakeSendFile(object):
def __init__(self, req):
self.req = req
def sendfile(self, o, i, offset, nbytes):
os.lseek(i, offset, os.SEEK_SET)
prev_len = len(self.req.body)
self.req.body += os.read(i, nbytes)
return len(self.req.body) - prev_len
class FakeGlanceConnection(object):
def __init__(self, *args, **kwargs):
self.sock = FakeSocket()
self.stub_force_sendfile = kwargs.get('stub_force_sendfile',
SENDFILE_SUPPORTED)
def connect(self):
return True
@ -75,9 +50,6 @@ def stub_out_store_server(stubs, base_dir, **kwargs):
def putrequest(self, method, url):
self.req = webob.Request.blank(url)
if self.stub_force_sendfile:
fake_sendfile = FakeSendFile(self.req)
stubs.Set(sendfile, 'sendfile', fake_sendfile.sendfile)
self.req.method = method
def putheader(self, key, value):
@ -118,15 +90,3 @@ def stub_out_store_server(stubs, base_dir, **kwargs):
def fake_image_iter(self):
for i in self.source.app_iter:
yield i
def fake_sendable(self, body):
force = getattr(self, 'stub_force_sendfile', None)
if force is None:
return self._stub_orig_sendable(body)
else:
if force:
assert glance.common.client.SENDFILE_SUPPORTED
return force
setattr(glance.common.client.BaseClient, '_stub_orig_sendable',
glance.common.client.BaseClient._sendable)

1
test-requirements.txt
View File

@ -20,7 +20,6 @@ boto3>=1.9.199 # Apache-2.0
# Optional packages that should be installed when testing
PyMySQL>=0.7.6 # MIT License
psycopg2>=2.8.4 # LGPL/ZPL
pysendfile>=2.0.0;sys_platform!='win32' # MIT
xattr>=0.9.2;sys_platform!='win32' # MIT
python-swiftclient>=3.2.0 # Apache-2.0
python-cinderclient>=4.1.0 # Apache-2.0