openstack
/
swift
Code Issues Proposed changes

Clean up protobuf message definitions 

Naming was inconsistent and some messages were reused, sometimes with
useless fields. Follow the grpc/protobuf best practice of naming input
and output protobuf message after the RPC function:
MyFunction() will take MyFunctionRequest() and return MyFunctionReply()

A couple of exceptions are made in cases where it does makes sense to
reuse a message (for listdir() like functions)

Change-Id: I5063cef634e2de80934c75aebed5707b1683d5d5
This commit is contained in:
Alexandre Lécuyer 2020-04-08 19:09:13 +02:00
parent 3dbcbee0e9
commit dd8e3a69c8
8 changed files with 3888 additions and 2962 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3227
go/swift-rpc-losf/proto/fmgr.pb.go
View File

File diff suppressed because it is too large Load Diff

80
go/swift-rpc-losf/rpc.go
View File

@ -85,7 +85,7 @@ type rpcFunc func(*server, context.Context, *[]byte) (*[]byte, error)
// RegisterVolume registers a new volume (volume) to the KV, given its index number and starting offset. // RegisterVolume registers a new volume (volume) to the KV, given its index number and starting offset.
// Will return an error if the volume index already exists. // Will return an error if the volume index already exists.
func RegisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func RegisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.NewVolumeInfo{} in := &pb.RegisterVolumeRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("RegisterVolume failed to unmarshal input: %v", err) logrus.Errorf("RegisterVolume failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -128,7 +128,7 @@ func RegisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, erro
} }
s.statsd_c.Increment("register_volume.ok") s.statsd_c.Increment("register_volume.ok")
out, err := proto.Marshal(&pb.NewVolumeReply{}) out, err := proto.Marshal(&pb.RegisterVolumeReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply for new volume entry: %v", err) reqlog.Errorf("failed to serialize reply for new volume entry: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply for new volume entry: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply for new volume entry: %v", err)
@ -138,7 +138,7 @@ func RegisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, erro
// UnregisterVolume will delete a volume entry from the kv. // UnregisterVolume will delete a volume entry from the kv.
func UnregisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func UnregisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.VolumeIndex{} in := &pb.UnregisterVolumeRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("UnregisterVolume failed to unmarshal input: %v", err) logrus.Errorf("UnregisterVolume failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -177,12 +177,12 @@ func UnregisterVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, er
} }
s.statsd_c.Increment("unregister_volume.ok") s.statsd_c.Increment("unregister_volume.ok")
return serializePb(&pb.NewVolumeReply{}) return serializePb(&pb.UnregisterVolumeReply{})
} }
// UpdateVolumeState will modify an existing volume state // UpdateVolumeState will modify an existing volume state
func UpdateVolumeState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func UpdateVolumeState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.NewVolumeState{} in := &pb.UpdateVolumeStateRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("UpdateVolumeState failed to unmarshal input: %v", err) logrus.Errorf("UpdateVolumeState failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -227,7 +227,7 @@ func UpdateVolumeState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, e
} }
s.statsd_c.Increment("update_volume_state.ok") s.statsd_c.Increment("update_volume_state.ok")
out, err := proto.Marshal(&pb.NewVolumeState{}) out, err := proto.Marshal(&pb.UpdateVolumeStateReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply for update volume: %v", err) reqlog.Errorf("failed to serialize reply for update volume: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply for update volume: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply for update volume: %v", err)
@ -237,7 +237,7 @@ func UpdateVolumeState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, e
// GetVolume will return a volume information // GetVolume will return a volume information
func GetVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func GetVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.VolumeIndex{} in := &pb.GetVolumeRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("GetVolume failed to unmarshal input: %v", err) logrus.Errorf("GetVolume failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -274,7 +274,7 @@ func GetVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
s.statsd_c.Increment("get_volume.ok") s.statsd_c.Increment("get_volume.ok")
pb_volume := pb.Volume{VolumeIndex: in.Index, VolumeType: pb.VolumeType(dfType), VolumeState: uint32(state), pb_volume := pb.GetVolumeReply{VolumeIndex: in.Index, VolumeType: pb.VolumeType(dfType), VolumeState: uint32(state),
Partition: uint32(partition), NextOffset: uint64(nextOffset)} Partition: uint32(partition), NextOffset: uint64(nextOffset)}
out, err := proto.Marshal(&pb_volume) out, err := proto.Marshal(&pb_volume)
if err != nil { if err != nil {
@ -289,7 +289,7 @@ func GetVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
// Currently this scans all volumes in the KV. Likely fast enough as long as the KV is cached. // Currently this scans all volumes in the KV. Likely fast enough as long as the KV is cached.
// If it becomes a performance issue, we may want to add an in-memory cache indexed by partition. // If it becomes a performance issue, we may want to add an in-memory cache indexed by partition.
func ListVolumes(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func ListVolumes(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ListVolumesInfo{} in := &pb.ListVolumesRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("ListVolumes failed to unmarshal input: %v", err) logrus.Errorf("ListVolumes failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -303,7 +303,7 @@ func ListVolumes(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error)
return nil, status.Errorf(codes.FailedPrecondition, "KV out of sync with volumes") return nil, status.Errorf(codes.FailedPrecondition, "KV out of sync with volumes")
} }
response := &pb.Volumes{} response := &pb.ListVolumesReply{}
// Iterate over volumes and return the ones that match the request // Iterate over volumes and return the ones that match the request
it := s.kv.NewIterator(volumePrefix) it := s.kv.NewIterator(volumePrefix)
@ -341,7 +341,7 @@ func ListVolumes(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error)
// RegisterObject registers a new object to the kv. // RegisterObject registers a new object to the kv.
func RegisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func RegisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.NewObjectInfo{} in := &pb.RegisterObjectRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("RegisterObject failed to unmarshal input: %v", err) logrus.Errorf("RegisterObject failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -426,7 +426,7 @@ func RegisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, erro
s.statsd_c.Increment("register_object.ok") s.statsd_c.Increment("register_object.ok")
out, err := proto.Marshal(&pb.NewObjectReply{}) out, err := proto.Marshal(&pb.RegisterObjectReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply: %v", err) reqlog.Errorf("failed to serialize reply: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err)
@ -436,7 +436,7 @@ func RegisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, erro
// UnregisterObject removes an an object entry from the kv. // UnregisterObject removes an an object entry from the kv.
func UnregisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func UnregisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ObjectName{} in := &pb.UnregisterObjectRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("UnregisterObject failed to unmarshal input: %v", err) logrus.Errorf("UnregisterObject failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -448,7 +448,7 @@ func UnregisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, er
}) })
reqlog.Debug("RPC Call") reqlog.Debug("RPC Call")
if !s.isClean { if !in.RepairTool && !s.isClean {
reqlog.Debug("KV out of sync with volumes") reqlog.Debug("KV out of sync with volumes")
return nil, status.Errorf(codes.FailedPrecondition, "KV out of sync with volumes") return nil, status.Errorf(codes.FailedPrecondition, "KV out of sync with volumes")
} }
@ -482,7 +482,7 @@ func UnregisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, er
} }
s.statsd_c.Increment("unregister_object.ok") s.statsd_c.Increment("unregister_object.ok")
out, err := proto.Marshal(&pb.DelObjectReply{}) out, err := proto.Marshal(&pb.UnregisterObjectReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply for del object reply: %v", err) reqlog.Errorf("failed to serialize reply for del object reply: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply for del object reply: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply for del object reply: %v", err)
@ -492,7 +492,7 @@ func UnregisterObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, er
// RenameObject changes an object key in the kv. (used for erasure code) // RenameObject changes an object key in the kv. (used for erasure code)
func RenameObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func RenameObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.RenameInfo{} in := &pb.RenameObjectRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -552,7 +552,7 @@ func RenameObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error)
s.statsd_c.Increment("rename_object.ok") s.statsd_c.Increment("rename_object.ok")
out, err := proto.Marshal(&pb.RenameReply{}) out, err := proto.Marshal(&pb.RenameObjectReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply: %v", err) reqlog.Errorf("failed to serialize reply: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err)
@ -562,7 +562,7 @@ func RenameObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error)
// LoadObject returns an object information // LoadObject returns an object information
func LoadObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func LoadObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.LoadObjectInfo{} in := &pb.LoadObjectRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -617,7 +617,7 @@ func LoadObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
s.statsd_c.Increment("load_object.ok") s.statsd_c.Increment("load_object.ok")
out, err := proto.Marshal(&pb.Object{Name: in.Name, VolumeIndex: volumeIndex, Offset: offset}) out, err := proto.Marshal(&pb.LoadObjectReply{Name: in.Name, VolumeIndex: volumeIndex, Offset: offset})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply: %v", err) reqlog.Errorf("failed to serialize reply: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err)
@ -627,7 +627,7 @@ func LoadObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
// QuarantineObject // QuarantineObject
func QuarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func QuarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ObjectName{} in := &pb.QuarantineObjectRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -681,7 +681,7 @@ func QuarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, er
s.statsd_c.Increment("quarantine_object.ok") s.statsd_c.Increment("quarantine_object.ok")
out, err := proto.Marshal(&pb.Empty{}) out, err := proto.Marshal(&pb.QuarantineObjectReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply: %v", err) reqlog.Errorf("failed to serialize reply: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err)
@ -691,7 +691,7 @@ func QuarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, er
// UnquarantineObject // UnquarantineObject
func UnquarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func UnquarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ObjectName{} in := &pb.UnquarantineObjectRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -742,7 +742,7 @@ func UnquarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte,
s.statsd_c.Increment("unquarantine_object.ok") s.statsd_c.Increment("unquarantine_object.ok")
out, err := proto.Marshal(&pb.Empty{}) out, err := proto.Marshal(&pb.UnquarantineObjectReply{})
if err != nil { if err != nil {
reqlog.Errorf("failed to serialize reply: %v", err) reqlog.Errorf("failed to serialize reply: %v", err)
return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err) return nil, status.Errorf(codes.Unavailable, "unable to serialize reply: %v", err)
@ -754,7 +754,7 @@ func UnquarantineObject(s *server, ctx context.Context, pbIn *[]byte) (*[]byte,
// In practice this is used to emulate the object hash directory that swift // In practice this is used to emulate the object hash directory that swift
// would create with the regular diskfile backend. // would create with the regular diskfile backend.
func LoadObjectsByPrefix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func LoadObjectsByPrefix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ObjectPrefix{} in := &pb.LoadObjectsByPrefixRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -789,7 +789,7 @@ func LoadObjectsByPrefix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte,
it := s.kv.NewIterator(objectPrefix) it := s.kv.NewIterator(objectPrefix)
defer it.Close() defer it.Close()
response := &pb.LoadObjectsResponse{} response := &pb.LoadObjectsByPrefixReply{}
// adds one byte because of prefix. Otherwise len(prefix) would be len(prefix)-1 // adds one byte because of prefix. Otherwise len(prefix) would be len(prefix)-1
for it.Seek(prefix); it.Valid() && len(prefix) <= len(it.Key()) && bytes.Equal(prefix, it.Key()[:len(prefix)]); it.Next() { for it.Seek(prefix); it.Valid() && len(prefix) <= len(it.Key()) && bytes.Equal(prefix, it.Key()[:len(prefix)]); it.Next() {
@ -824,7 +824,7 @@ func LoadObjectsByPrefix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte,
// PageSize is the maximum count of items to return. If zero, the server will pick a reasonnable limit. // PageSize is the maximum count of items to return. If zero, the server will pick a reasonnable limit.
// func (s *server) LoadObjectsByVolume(in *pb.VolumeIndex, stream pb.FileMgr_LoadObjectsByVolumeServer) error { // func (s *server) LoadObjectsByVolume(in *pb.VolumeIndex, stream pb.FileMgr_LoadObjectsByVolumeServer) error {
func LoadObjectsByVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func LoadObjectsByVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.VolumeIndex{} in := &pb.LoadObjectsByVolumeRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -871,7 +871,7 @@ func LoadObjectsByVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte,
} }
defer it.Close() defer it.Close()
response := &pb.LoadObjectsResponse{} response := &pb.LoadObjectsByVolumeReply{}
// Objects are not indexed by volume. We have to scan the whole KV and examine each value. // Objects are not indexed by volume. We have to scan the whole KV and examine each value.
// It shouldn't matter as this is only used for compaction, and each object will have to be copied. // It shouldn't matter as this is only used for compaction, and each object will have to be copied.
@ -929,7 +929,7 @@ func LoadObjectsByVolume(s *server, ctx context.Context, pbIn *[]byte) (*[]byte,
// ListPartitions returns a list of partitions for which we have objects. // ListPartitions returns a list of partitions for which we have objects.
// This is used to emulate a listdir() of partitions below the "objects" directory. // This is used to emulate a listdir() of partitions below the "objects" directory.
func ListPartitions(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func ListPartitions(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ListPartitionsInfo{} in := &pb.ListPartitionsRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -1047,7 +1047,7 @@ func ListPartitions(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, erro
// ListPartition returns a list of suffixes within a partition // ListPartition returns a list of suffixes within a partition
func ListPartition(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func ListPartition(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ListPartitionInfo{} in := &pb.ListPartitionRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -1135,7 +1135,7 @@ func ListPartition(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error
// ListSuffix returns a list of object hashes below the partition and suffix // ListSuffix returns a list of object hashes below the partition and suffix
func ListSuffix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func ListSuffix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ListSuffixInfo{} in := &pb.ListSuffixRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -1242,7 +1242,7 @@ func ListSuffix(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
// object hash. PageSize is the maximum count of items to return. If zero, // object hash. PageSize is the maximum count of items to return. If zero,
// the server will pick a reasonnable limit. // the server will pick a reasonnable limit.
func ListQuarantinedOHashes(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func ListQuarantinedOHashes(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ListQuarantinedOHashesInfo{} in := &pb.ListQuarantinedOHashesRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -1287,7 +1287,7 @@ func ListQuarantinedOHashes(s *server, ctx context.Context, pbIn *[]byte) (*[]by
it := s.kv.NewIterator(quarantinePrefix) it := s.kv.NewIterator(quarantinePrefix)
defer it.Close() defer it.Close()
response := &pb.QuarantinedObjectNames{} response := &pb.ListQuarantinedOHashesReply{}
curKey := make([]byte, maxObjKeyLen) curKey := make([]byte, maxObjKeyLen)
lastOhash := make([]byte, 32) lastOhash := make([]byte, 32)
@ -1331,7 +1331,7 @@ func ListQuarantinedOHashes(s *server, ctx context.Context, pbIn *[]byte) (*[]by
} }
func ListQuarantinedOHash(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func ListQuarantinedOHash(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.ObjectPrefix{} in := &pb.ListQuarantinedOHashRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -1364,7 +1364,7 @@ func ListQuarantinedOHash(s *server, ctx context.Context, pbIn *[]byte) (*[]byte
it := s.kv.NewIterator(quarantinePrefix) it := s.kv.NewIterator(quarantinePrefix)
defer it.Close() defer it.Close()
response := &pb.LoadObjectsResponse{} response := &pb.ListQuarantinedOHashReply{}
// adds one byte because of prefix. Otherwise len(prefix) would be len(prefix)-1 // adds one byte because of prefix. Otherwise len(prefix) would be len(prefix)-1
for it.Seek(prefix); it.Valid() && len(prefix) <= len(it.Key()) && bytes.Equal(prefix, it.Key()[:len(prefix)]); it.Next() { for it.Seek(prefix); it.Valid() && len(prefix) <= len(it.Key()) && bytes.Equal(prefix, it.Key()[:len(prefix)]); it.Next() {
@ -1392,7 +1392,7 @@ func ListQuarantinedOHash(s *server, ctx context.Context, pbIn *[]byte) (*[]byte
} }
func GetNextOffset(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func GetNextOffset(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.GetNextOffsetInfo{} in := &pb.GetNextOffsetRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
@ -1429,18 +1429,18 @@ func GetNextOffset(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error
} }
s.statsd_c.Increment("get_next_offset.ok") s.statsd_c.Increment("get_next_offset.ok")
return serializePb(&pb.VolumeNextOffset{Offset: uint64(nextOffset)}) return serializePb(&pb.GetNextOffsetReply{Offset: uint64(nextOffset)})
} }
// GetStats returns stats for the KV. used for initial debugging, remove? // GetStats returns stats for the KV. used for initial debugging, remove?
func GetStats(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func GetStats(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.GetStatsInfo{} in := &pb.GetStatsRequest{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")
} }
response := new(pb.KVStats) response := new(pb.GetStatsReply)
m := CollectStats(s) m := CollectStats(s)
response.Stats = m response.Stats = m
@ -1460,12 +1460,12 @@ func SetKvState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
reqlog.Debug("RPC Call") reqlog.Debug("RPC Call")
s.isClean = in.IsClean s.isClean = in.IsClean
return serializePb(&pb.Empty{}) return serializePb(&pb.SetKvStateReply{})
} }
// Gets KV state (is in sync with volumes, or not) // Gets KV state (is in sync with volumes, or not)
func GetKvState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) { func GetKvState(s *server, ctx context.Context, pbIn *[]byte) (*[]byte, error) {
in := &pb.Empty{} in := &pb.KvState{}
if err := proto.Unmarshal(*pbIn, in); err != nil { if err := proto.Unmarshal(*pbIn, in); err != nil {
logrus.Errorf("failed to unmarshal input: %v", err) logrus.Errorf("failed to unmarshal input: %v", err)
return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf") return nil, status.Errorf(codes.InvalidArgument, "unable to deserialize protobuf")

72
go/swift-rpc-losf/rpc_test.go
View File

@ -84,7 +84,7 @@ func check_200(response *http.Response, err error) error {
} }
func populateKV() (err error) { func populateKV() (err error) {
volumes := []pb.NewVolumeInfo{ volumes := []pb.RegisterVolumeRequest{
{Partition: 9, Type: 0, VolumeIndex: 20, Offset: 8192, State: 0, RepairTool: false}, {Partition: 9, Type: 0, VolumeIndex: 20, Offset: 8192, State: 0, RepairTool: false},
{Partition: 10, Type: 0, VolumeIndex: 35, Offset: 8192, State: 0, RepairTool: false}, {Partition: 10, Type: 0, VolumeIndex: 35, Offset: 8192, State: 0, RepairTool: false},
{Partition: 40, Type: 0, VolumeIndex: 24, Offset: 8192, State: 0, RepairTool: false}, {Partition: 40, Type: 0, VolumeIndex: 24, Offset: 8192, State: 0, RepairTool: false},
@ -137,7 +137,7 @@ func populateKV() (err error) {
{Partition: 1019, Type: 0, VolumeIndex: 10, Offset: 8192, State: 0, RepairTool: false}, {Partition: 1019, Type: 0, VolumeIndex: 10, Offset: 8192, State: 0, RepairTool: false},
} }
objects := []pb.NewObjectInfo{ objects := []pb.RegisterObjectRequest{
{Name: []byte("85fd12f8961e33cbf7229a94118524fa1515589781.45671.ts"), VolumeIndex: 3, Offset: 8192, NextOffset: 12288, RepairTool: false}, {Name: []byte("85fd12f8961e33cbf7229a94118524fa1515589781.45671.ts"), VolumeIndex: 3, Offset: 8192, NextOffset: 12288, RepairTool: false},
{Name: []byte("84afc1659c7e8271951fe370d6eee0f81515590332.51834.ts"), VolumeIndex: 5, Offset: 4096, NextOffset: 8192, RepairTool: false}, {Name: []byte("84afc1659c7e8271951fe370d6eee0f81515590332.51834.ts"), VolumeIndex: 5, Offset: 4096, NextOffset: 8192, RepairTool: false},
{Name: []byte("f45bf9000f39092b9de5a74256e3eebe1515590648.06511.ts"), VolumeIndex: 7, Offset: 4096, NextOffset: 8192, RepairTool: false}, {Name: []byte("f45bf9000f39092b9de5a74256e3eebe1515590648.06511.ts"), VolumeIndex: 7, Offset: 4096, NextOffset: 8192, RepairTool: false},
@ -270,7 +270,7 @@ func TestMain(m *testing.M) {
// - single object // - single object
// - first and last elements of the KV // - first and last elements of the KV
func TestLoadObjectsByPrefix(t *testing.T) { func TestLoadObjectsByPrefix(t *testing.T) {
prefix := &pb.ObjectPrefix{Prefix: []byte("105de5f388ab4b72e56bc93f36ad388a")} prefix := &pb.LoadObjectsByPrefixRequest{Prefix: []byte("105de5f388ab4b72e56bc93f36ad388a")}
out, err := proto.Marshal(prefix) out, err := proto.Marshal(prefix)
if err != nil { if err != nil {
@ -289,7 +289,7 @@ func TestLoadObjectsByPrefix(t *testing.T) {
} }
defer response.Body.Close() defer response.Body.Close()
r := &pb.LoadObjectsResponse{} r := &pb.LoadObjectsByPrefixReply{}
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -307,7 +307,7 @@ func TestLoadObjectsByPrefix(t *testing.T) {
func TestListPartitions(t *testing.T) { func TestListPartitions(t *testing.T) {
partPower := uint32(10) partPower := uint32(10)
lpInfo := &pb.ListPartitionsInfo{PartitionBits: partPower} lpInfo := &pb.ListPartitionsRequest{PartitionBits: partPower}
out, err := proto.Marshal(lpInfo) out, err := proto.Marshal(lpInfo)
if err != nil { if err != nil {
t.Error("failed to marshal") t.Error("failed to marshal")
@ -346,7 +346,7 @@ func TestListSuffix(t *testing.T) {
partPower := uint32(10) partPower := uint32(10)
suffix := []byte("845") suffix := []byte("845")
lsInfo := &pb.ListSuffixInfo{Partition: partition, Suffix: suffix, PartitionBits: partPower} lsInfo := &pb.ListSuffixRequest{Partition: partition, Suffix: suffix, PartitionBits: partPower}
out, err := proto.Marshal(lsInfo) out, err := proto.Marshal(lsInfo)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
@ -394,7 +394,7 @@ func TestState(t *testing.T) {
} }
response.Body.Close() response.Body.Close()
empty := &pb.Empty{} empty := &pb.GetKvStateRequest{}
empty_serialized, err := proto.Marshal(empty) empty_serialized, err := proto.Marshal(empty)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
@ -452,7 +452,7 @@ func TestState(t *testing.T) {
func TestRegisterObject(t *testing.T) { func TestRegisterObject(t *testing.T) {
// Register new non-existing object // Register new non-existing object
name := []byte("33dea50d391ee52a8ead7cb562a9b4e2/1539791765.84449#5#d.data") name := []byte("33dea50d391ee52a8ead7cb562a9b4e2/1539791765.84449#5#d.data")
obj := &pb.NewObjectInfo{Name: name, VolumeIndex: 1, Offset: 4096, NextOffset: 8192, RepairTool: false} obj := &pb.RegisterObjectRequest{Name: name, VolumeIndex: 1, Offset: 4096, NextOffset: 8192, RepairTool: false}
out, err := proto.Marshal(obj) out, err := proto.Marshal(obj)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -465,7 +465,7 @@ func TestRegisterObject(t *testing.T) {
} }
response.Body.Close() response.Body.Close()
objInfo := &pb.LoadObjectInfo{Name: name, IsQuarantined: false, RepairTool: false} objInfo := &pb.LoadObjectRequest{Name: name, IsQuarantined: false, RepairTool: false}
out, err = proto.Marshal(objInfo) out, err = proto.Marshal(objInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -488,7 +488,7 @@ func TestRegisterObject(t *testing.T) {
} }
// Register existing object, which should fail // Register existing object, which should fail
obj = &pb.NewObjectInfo{Name: name, VolumeIndex: 1, Offset: 4096, NextOffset: 8192, RepairTool: false} obj = &pb.RegisterObjectRequest{Name: name, VolumeIndex: 1, Offset: 4096, NextOffset: 8192, RepairTool: false}
out, err = proto.Marshal(obj) out, err = proto.Marshal(obj)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -504,7 +504,7 @@ func TestRegisterObject(t *testing.T) {
response.Body.Close() response.Body.Close()
// Remove object // Remove object
unregInfo := &pb.ObjectName{Name: name} unregInfo := &pb.UnregisterObjectRequest{Name: name}
out, err = proto.Marshal(unregInfo) out, err = proto.Marshal(unregInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -530,7 +530,7 @@ func TestRegisterObject(t *testing.T) {
func TestQuarantineObject(t *testing.T) { func TestQuarantineObject(t *testing.T) {
// Quarantine an existing object // Quarantine an existing object
name := []byte("bf43763a98208f15da803e76bf52e7d11515750803.01357#0#d.data") name := []byte("bf43763a98208f15da803e76bf52e7d11515750803.01357#0#d.data")
objName := &pb.ObjectName{Name: name, RepairTool: false} objName := &pb.QuarantineObjectRequest{Name: name, RepairTool: false}
out, err := proto.Marshal(objName) out, err := proto.Marshal(objName)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -543,7 +543,7 @@ func TestQuarantineObject(t *testing.T) {
response.Body.Close() response.Body.Close()
// We shouldn't be able to find it // We shouldn't be able to find it
objInfo := &pb.LoadObjectInfo{Name: name, IsQuarantined: false, RepairTool: false} objInfo := &pb.LoadObjectRequest{Name: name, IsQuarantined: false, RepairTool: false}
out, err = proto.Marshal(objInfo) out, err = proto.Marshal(objInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -565,7 +565,7 @@ func TestQuarantineObject(t *testing.T) {
func TestUnquarantineObject(t *testing.T) { func TestUnquarantineObject(t *testing.T) {
// Unquarantine an existing quarantined object (check that) // Unquarantine an existing quarantined object (check that)
name := []byte("bf43763a98208f15da803e76bf52e7d11515750803.01357#0#d.data") name := []byte("bf43763a98208f15da803e76bf52e7d11515750803.01357#0#d.data")
objName := &pb.ObjectName{Name: name, RepairTool: false} objName := &pb.UnquarantineObjectRequest{Name: name, RepairTool: false}
out, err := proto.Marshal(objName) out, err := proto.Marshal(objName)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -579,7 +579,7 @@ func TestUnquarantineObject(t *testing.T) {
response.Body.Close() response.Body.Close()
// We should be able to find it // We should be able to find it
objInfo := &pb.LoadObjectInfo{Name: name, IsQuarantined: false, RepairTool: false} objInfo := &pb.LoadObjectRequest{Name: name, IsQuarantined: false, RepairTool: false}
out, err = proto.Marshal(objInfo) out, err = proto.Marshal(objInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -599,7 +599,7 @@ func TestUnquarantineObject(t *testing.T) {
// This test modifies the DB // This test modifies the DB
func TestListQuarantinedOHashes(t *testing.T) { func TestListQuarantinedOHashes(t *testing.T) {
// We shouldn't find any quarantined object initially // We shouldn't find any quarantined object initially
lqInfo := &pb.ListQuarantinedOHashesInfo{PageSize: 100} lqInfo := &pb.ListQuarantinedOHashesRequest{PageSize: 100}
out, err := proto.Marshal(lqInfo) out, err := proto.Marshal(lqInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -611,7 +611,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
t.Fatalf("failed to list quarantined ohashes: %v", err) t.Fatalf("failed to list quarantined ohashes: %v", err)
} }
r := &pb.QuarantinedObjectNames{} r := &pb.ListQuarantinedOHashesReply{}
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -624,7 +624,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
response.Body.Close() response.Body.Close()
// Quarantine a few objects and check we can find them // Quarantine a few objects and check we can find them
objectsToQuarantine := []pb.ObjectName{ objectsToQuarantine := []pb.QuarantineObjectRequest{
{Name: []byte("02573d31b770cda8e0effd7762e8a0751515750801.09785#2#d.data"), RepairTool: false}, {Name: []byte("02573d31b770cda8e0effd7762e8a0751515750801.09785#2#d.data"), RepairTool: false},
{Name: []byte("6b08eabf5667557c72dc6570aa1fb8451515750801.08639#4#d.data"), RepairTool: false}, {Name: []byte("6b08eabf5667557c72dc6570aa1fb8451515750801.08639#4#d.data"), RepairTool: false},
{Name: []byte("6b08eabf5667557c72dc6570aa1fb8451515750856.77219.meta"), RepairTool: false}, {Name: []byte("6b08eabf5667557c72dc6570aa1fb8451515750856.77219.meta"), RepairTool: false},
@ -658,7 +658,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
} }
// List quarantined objects // List quarantined objects
lqInfo = &pb.ListQuarantinedOHashesInfo{PageSize: 100} lqInfo = &pb.ListQuarantinedOHashesRequest{PageSize: 100}
out, err = proto.Marshal(lqInfo) out, err = proto.Marshal(lqInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -670,7 +670,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
t.Fatalf("failed to list quarantined ohashes: %v", err) t.Fatalf("failed to list quarantined ohashes: %v", err)
} }
r = &pb.QuarantinedObjectNames{} r = &pb.ListQuarantinedOHashesReply{}
buf.Reset() buf.Reset()
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -694,7 +694,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
} }
// List quarantined objects, with a PageSize of 1 // List quarantined objects, with a PageSize of 1
lqInfo = &pb.ListQuarantinedOHashesInfo{PageSize: 1} lqInfo = &pb.ListQuarantinedOHashesRequest{PageSize: 1}
out, err = proto.Marshal(lqInfo) out, err = proto.Marshal(lqInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -706,7 +706,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
t.Fatalf("failed to list quarantined ohashes: %v", err) t.Fatalf("failed to list quarantined ohashes: %v", err)
} }
r = &pb.QuarantinedObjectNames{} r = &pb.ListQuarantinedOHashesReply{}
buf.Reset() buf.Reset()
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -730,7 +730,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
} }
// Get the next two entries // Get the next two entries
lqInfo = &pb.ListQuarantinedOHashesInfo{PageSize: 2, PageToken: r.NextPageToken} lqInfo = &pb.ListQuarantinedOHashesRequest{PageSize: 2, PageToken: r.NextPageToken}
out, err = proto.Marshal(lqInfo) out, err = proto.Marshal(lqInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -742,7 +742,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
t.Fatalf("failed to list quarantined ohashes: %v", err) t.Fatalf("failed to list quarantined ohashes: %v", err)
} }
r = &pb.QuarantinedObjectNames{} r = &pb.ListQuarantinedOHashesReply{}
buf.Reset() buf.Reset()
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -766,7 +766,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
} }
// Get all remaining entries // Get all remaining entries
lqInfo = &pb.ListQuarantinedOHashesInfo{PageSize: 100, PageToken: r.NextPageToken} lqInfo = &pb.ListQuarantinedOHashesRequest{PageSize: 100, PageToken: r.NextPageToken}
out, err = proto.Marshal(lqInfo) out, err = proto.Marshal(lqInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -778,7 +778,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
t.Fatalf("failed to list quarantined ohashes: %v", err) t.Fatalf("failed to list quarantined ohashes: %v", err)
} }
r = &pb.QuarantinedObjectNames{} r = &pb.ListQuarantinedOHashesReply{}
buf.Reset() buf.Reset()
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -805,7 +805,7 @@ func TestListQuarantinedOHashes(t *testing.T) {
func TestLoadObjectsByVolume(t *testing.T) { func TestLoadObjectsByVolume(t *testing.T) {
// List non quarantined objects from volume 22, we should not find any // List non quarantined objects from volume 22, we should not find any
volIndex := &pb.VolumeIndex{Index: 22} volIndex := &pb.LoadObjectsByVolumeRequest{Index: 22}
out, err := proto.Marshal(volIndex) out, err := proto.Marshal(volIndex)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -817,7 +817,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
t.Fatalf("failed to call LoadObjectsByVolume: %v", err) t.Fatalf("failed to call LoadObjectsByVolume: %v", err)
} }
r := &pb.LoadObjectsResponse{} r := &pb.LoadObjectsByVolumeReply{}
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -829,7 +829,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
} }
// List quarantined objects from volume 22 // List quarantined objects from volume 22
volIndex = &pb.VolumeIndex{Index: 22, Quarantined: true} volIndex = &pb.LoadObjectsByVolumeRequest{Index: 22, Quarantined: true}
out, err = proto.Marshal(volIndex) out, err = proto.Marshal(volIndex)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -841,7 +841,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
t.Fatalf("failed to call LoadObjectsByVolume: %v", err) t.Fatalf("failed to call LoadObjectsByVolume: %v", err)
} }
r = &pb.LoadObjectsResponse{} r = &pb.LoadObjectsByVolumeReply{}
buf = new(bytes.Buffer) buf = new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -875,7 +875,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
} }
// List quarantined objects from volume 22 with pagination // List quarantined objects from volume 22 with pagination
volIndex = &pb.VolumeIndex{Index: 22, Quarantined: true, PageSize: 1} volIndex = &pb.LoadObjectsByVolumeRequest{Index: 22, Quarantined: true, PageSize: 1}
out, err = proto.Marshal(volIndex) out, err = proto.Marshal(volIndex)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -887,7 +887,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
t.Fatalf("failed to call LoadObjectsByVolume: %v", err) t.Fatalf("failed to call LoadObjectsByVolume: %v", err)
} }
r = &pb.LoadObjectsResponse{} r = &pb.LoadObjectsByVolumeReply{}
buf = new(bytes.Buffer) buf = new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -914,7 +914,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
} }
// Second call with pagination // Second call with pagination
volIndex = &pb.VolumeIndex{Index: 22, Quarantined: true, PageSize: 1, PageToken: r.NextPageToken} volIndex = &pb.LoadObjectsByVolumeRequest{Index: 22, Quarantined: true, PageSize: 1, PageToken: r.NextPageToken}
out, err = proto.Marshal(volIndex) out, err = proto.Marshal(volIndex)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -926,7 +926,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
t.Fatalf("failed to call LoadObjectsByVolume: %v", err) t.Fatalf("failed to call LoadObjectsByVolume: %v", err)
} }
r = &pb.LoadObjectsResponse{} r = &pb.LoadObjectsByVolumeReply{}
buf = new(bytes.Buffer) buf = new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), r); err != nil { if err = proto.Unmarshal(buf.Bytes(), r); err != nil {
@ -953,7 +953,7 @@ func TestLoadObjectsByVolume(t *testing.T) {
} }
func TestListQuarantinedOHash(t *testing.T) { func TestListQuarantinedOHash(t *testing.T) {
ohash := &pb.ObjectPrefix{Prefix: []byte("6b08eabf5667557c72dc6570aa1fb845"), RepairTool: false} ohash := &pb.ListQuarantinedOHashRequest{Prefix: []byte("6b08eabf5667557c72dc6570aa1fb845"), RepairTool: false}
out, err := proto.Marshal(ohash) out, err := proto.Marshal(ohash)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -965,7 +965,7 @@ func TestListQuarantinedOHash(t *testing.T) {
t.Fatalf("error listing quarantined object files: %s", err) t.Fatalf("error listing quarantined object files: %s", err)
} }
qList := &pb.LoadObjectsResponse{} qList := &pb.ListQuarantinedOHashReply{}
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
buf.ReadFrom(response.Body) buf.ReadFrom(response.Body)
if err = proto.Unmarshal(buf.Bytes(), qList); err != nil { if err = proto.Unmarshal(buf.Bytes(), qList); err != nil {

370
swift/obj/fmgr.proto
View File

@ -5,39 +5,186 @@ package filemgr;
// Python: protoc -I. --python_out=. fmgr.proto // Python: protoc -I. --python_out=. fmgr.proto
// Golang : protoc -I proto proto/fmgr.proto --go_out=proto // Golang : protoc -I proto proto/fmgr.proto --go_out=proto
message ListPartitionsInfo { message RegisterVolumeRequest {
uint32 partition = 1; // Swift partition
VolumeType type = 2;
uint32 volume_index = 3;
uint64 offset = 4; // Next available offset to use in the volume.
VolumeState state = 5;
bool repair_tool = 6; // Request is coming from a repair tool
}
message RegisterVolumeReply {}
message UnregisterVolumeRequest {
uint32 index = 1;
bool repair_tool = 2;
}
message UnregisterVolumeReply {}
message UpdateVolumeStateRequest {
uint32 volume_index = 1;
VolumeState state = 2;
bool repair_tool = 3;
}
message UpdateVolumeStateReply {}
message GetVolumeRequest {
uint32 index = 1;
bool repair_tool = 2;
}
message GetVolumeReply {
uint32 volume_index = 1;
VolumeType volume_type = 2;
uint32 volume_state = 3;
uint32 partition = 4;
uint64 next_offset = 5;
}
message ListVolumesRequest {
uint32 partition = 1;
VolumeType type = 2;
bool repair_tool = 3;
}
message ListVolumesReply {
repeated Volume volumes = 1;
}
message RegisterObjectRequest {
bytes name = 1;
uint32 volume_index = 2;
uint64 offset = 3; // Object offset within volume
uint64 next_offset = 4; // Next offset to start from in the volume
bool repair_tool = 5;
}
message RegisterObjectReply {}
message UnregisterObjectRequest {
bytes name = 1;
bool repair_tool = 2;
}
message UnregisterObjectReply {}
message RenameObjectRequest {
bytes name = 1;
bytes new_name = 2;
bool repair_tool = 3;
}
message RenameObjectReply {}
message LoadObjectRequest {
bytes name = 1;
bool is_quarantined = 2;
bool repair_tool = 3;
}
message LoadObjectReply {
bytes name = 1;
uint32 volume_index = 2;
uint64 offset = 3;
}
message QuarantineObjectRequest {
bytes name = 1;
bool repair_tool = 2;
}
message QuarantineObjectReply {}
message UnquarantineObjectRequest {
bytes name = 1;
bool repair_tool = 2;
}
message UnquarantineObjectReply {}
message LoadObjectsByPrefixRequest {
bytes prefix = 1;
bool repair_tool = 2;
}
message LoadObjectsByPrefixReply {
repeated Object objects = 1;
}
message LoadObjectsByVolumeRequest {
uint32 index = 1;
bool quarantined = 2; // List only quarantined files, if true
bytes page_token = 3;
uint32 page_size = 4;
bool repair_tool = 5;
}
message LoadObjectsByVolumeReply {
repeated Object objects = 1;
bytes next_page_token = 2;
}
message ListPartitionsRequest {
uint32 partition_bits = 1; uint32 partition_bits = 1;
} }
message ListPartitionInfo { message ListPartitionRequest {
uint32 partition = 1; uint32 partition = 1;
uint32 partition_bits = 2; uint32 partition_bits = 2;
} }
message ListSuffixInfo { message ListSuffixRequest {
uint32 partition = 1; uint32 partition = 1;
bytes suffix = 2; bytes suffix = 2;
uint32 partition_bits = 3; uint32 partition_bits = 3;
} }
// Generic reply message for List* functions (listdir like) message ListQuarantinedOHashesRequest {
message DirEntries { bytes page_token = 1;
repeated string entry = 1; uint32 page_size = 2;
} }
message VolumeIndex { message ListQuarantinedOHashesReply {
uint32 index = 1; repeated QuarantinedObjectName objects = 1;
bytes next_page_token = 2;
// List only quarantined files, if true
bool quarantined = 2;
bytes page_token = 3;
uint32 page_size = 4;
// Is this request coming from a repair tool ?
bool repair_tool = 5;
} }
message ListQuarantinedOHashRequest {
bytes prefix = 1;
bool repair_tool = 2;
}
message ListQuarantinedOHashReply {
repeated Object objects = 1;
}
message GetNextOffsetRequest {
uint32 volume_index = 1;
bool repair_tool = 2;
}
message GetNextOffsetReply {
uint64 offset = 1;
}
message GetStatsRequest {}
message GetStatsReply {
map<string, uint64> stats = 1;
}
message SetKvStateReply {}
message GetKvStateRequest {}
message KvState {
bool isClean = 1;
}
// Generic messages
message Volume { message Volume {
uint32 volume_index = 1; uint32 volume_index = 1;
VolumeType volume_type = 2; VolumeType volume_type = 2;
@ -46,43 +193,28 @@ message Volume {
uint64 next_offset = 5; uint64 next_offset = 5;
} }
// Similar to volume but we don't want to require the message Object {
// partition bytes name = 1;
message GetNextOffsetInfo { uint32 volume_index = 2;
uint32 volume_index = 1; uint64 offset = 3;
uint32 volume_type = 2;
uint32 volume_state = 3;
// Is this request coming from a repair tool ?
bool repair_tool = 4;
} }
message ListVolumesInfo { message QuarantinedObjectName {
uint32 partition = 1; bytes name = 1;
VolumeType type = 2;
// Is this request coming from a repair tool ?
bool repair_tool = 3;
} }
message Volumes { // For listdir() like functions
repeated Volume volumes = 1; message DirEntries {
repeated string entry = 1;
} }
// The response message to GetNextOffset // Enums
message VolumeNextOffset {
uint64 offset = 1;
}
// Volume type
enum VolumeType { enum VolumeType {
VOLUME_DEFAULT = 0; VOLUME_DEFAULT = 0;
VOLUME_TOMBSTONE = 1; VOLUME_TOMBSTONE = 1;
VOLUME_X_DELETE_AT = 2; VOLUME_X_DELETE_AT = 2;
} }
// VolumeState state
enum VolumeState { enum VolumeState {
// Default state, volume can be read from and written to // Default state, volume can be read from and written to
STATE_RW = 0; STATE_RW = 0;
@ -91,157 +223,3 @@ enum VolumeState {
// Volume is a compaction target. New objects cannot be appended // Volume is a compaction target. New objects cannot be appended
STATE_COMPACTION_TARGET = 2; STATE_COMPACTION_TARGET = 2;
} }
// The request message for a new volume
message NewVolumeInfo {
// Swift partition
uint32 partition = 1;
VolumeType type = 2;
// Index number of the volume. It is up to the client to map this to an actual filename.
uint32 volume_index = 3;
// Next available offset to use in the volume.
uint64 offset = 4;
VolumeState state = 5;
// Is this request coming from a repair tool ?
bool repair_tool = 6;
}
message NewVolumeState {
uint32 volume_index = 1;
VolumeState state = 2;
// Is this request coming from a repair tool ?
bool repair_tool = 3;
}
// The response message for a new volume
message NewVolumeReply {
}
message DelObjectReply {
}
message RenameReply {
}
// The request message for a new object
message NewObjectInfo {
// Object "name". Name made of (md5, timestamp, etc..)
bytes name = 1;
// Index number of the volume.
uint32 volume_index = 2;
// Start offset of the object in the volume.
uint64 offset = 3;
// Next available offset to use in the volume.
uint64 next_offset = 4;
// Is this request coming from a repair tool ?
bool repair_tool = 5;
}
// The response message for a new object
// Currently empty, but we may want to return something in the future
message NewObjectReply {
}
message QuarantinedObjectName {
bytes name = 1;
}
message QuarantinedObjectNames {
repeated QuarantinedObjectName objects = 1;
bytes next_page_token = 2;
}
message ObjectName {
// name of the object.
bytes name = 1;
// Is this request coming from a repair tool ?
bool repair_tool = 2;
}
message LoadObjectInfo {
// name of the object.
bytes name = 1;
// Is it quarantined ?
bool is_quarantined = 2;
// Is this request coming from a repair tool ?
bool repair_tool = 5;
}
// The request message to rename an object
message RenameInfo {
// name of the object.
bytes name = 1;
// new name of the object
bytes new_name = 2;
// Is this request coming from a repair tool ?
bool repair_tool = 3;
}
message Object {
// name of the object.
bytes name = 1;
// Index number of the volume.
uint32 volume_index = 2;
// Start offset of the object in the volume.
uint64 offset = 3;
}
message LoadObjectsResponse {
repeated Object objects = 1;
bytes next_page_token = 2;
}
message ObjectPrefix {
bytes prefix = 1;
// Is this request coming from a repair tool ?
bool repair_tool = 2;
}
message Empty {
}
message GetStatsInfo {
}
message PartitionContent {
repeated FullPathEntry file_entries = 1;
}
message FullPathEntry {
// We could add the partition if needed
bytes suffix = 1;
bytes ohash = 2; // Object hash
bytes filename = 3; // '.data', '.meta'.. files
}
message KvState {
bool isClean = 1;
}
// KV stats
message KVStats {
map<string, uint64> stats = 1;
}
message ListQuarantinedOHashesInfo {
bytes page_token = 1;
uint32 page_size = 2;
}

2944
swift/obj/fmgr_pb2.py
View File

File diff suppressed because one or more lines are too long

149
swift/obj/rpc_http.py
View File

@ -22,6 +22,7 @@ from swift.obj import fmgr_pb2 as pb
class UnixHTTPConnection(httplib.HTTPConnection): class UnixHTTPConnection(httplib.HTTPConnection):
"""Support for unix domain socket with httplib""" """Support for unix domain socket with httplib"""
def __init__(self, path, host='localhost', port=None, strict=None, def __init__(self, path, host='localhost', port=None, strict=None,
timeout=None): timeout=None):
httplib.HTTPConnection.__init__(self, host, port=port, strict=strict, httplib.HTTPConnection.__init__(self, host, port=port, strict=strict,
@ -76,43 +77,45 @@ def get_next_offset(socket_path, volume_index, repair_tool=False):
""" """
Returns the next offset to use in the volume Returns the next offset to use in the volume
""" """
volume = pb.GetNextOffsetInfo(volume_index=int(volume_index), volume = pb.GetNextOffsetRequest(volume_index=int(volume_index),
repair_tool=repair_tool) repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/get_next_offset', volume.SerializeToString()) conn.request('POST', '/get_next_offset', volume.SerializeToString())
response = get_rpc_reply(conn, pb.VolumeNextOffset) response = get_rpc_reply(conn, pb.GetNextOffsetReply)
return response.offset return response.offset
def register_volume(socket_path, partition, volume_type, volume_index, def register_volume(socket_path, partition, volume_type, volume_index,
first_obj_offset, state, repair_tool=False): first_obj_offset, state, repair_tool=False):
volume = pb.NewVolumeInfo(partition=int(partition), type=int(volume_type), volume = pb.RegisterVolumeRequest(partition=int(partition),
volume_index=int(volume_index), type=int(volume_type),
offset=first_obj_offset, state=state, volume_index=int(volume_index),
repair_tool=repair_tool) offset=first_obj_offset, state=state,
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/register_volume', volume.SerializeToString()) conn.request('POST', '/register_volume', volume.SerializeToString())
response = get_rpc_reply(conn, pb.NewVolumeReply) response = get_rpc_reply(conn, pb.RegisterVolumeReply)
return response return response
def unregister_volume(socket_path, volume_index): def unregister_volume(socket_path, volume_index):
index = pb.VolumeIndex(index=volume_index) index = pb.UnregisterVolumeRequest(index=volume_index)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/unregister_volume', index.SerializeToString()) conn.request('POST', '/unregister_volume', index.SerializeToString())
response = get_rpc_reply(conn, pb.Empty) response = get_rpc_reply(conn, pb.UnregisterVolumeReply)
return response return response
def update_volume_state(socket_path, volume_index, new_state, def update_volume_state(socket_path, volume_index, new_state,
repair_tool=False): repair_tool=False):
state_update = pb.NewVolumeState(volume_index=int(volume_index), state_update = pb.UpdateVolumeStateRequest(volume_index=int(volume_index),
state=new_state, repair_tool=repair_tool) state=new_state,
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/update_volume_state', conn.request('POST', '/update_volume_state',
state_update.SerializeToString()) state_update.SerializeToString())
response = get_rpc_reply(conn, pb.Empty) response = get_rpc_reply(conn, pb.UpdateVolumeStateReply)
return response return response
@ -121,46 +124,49 @@ def register_object(socket_path, name, volume_index, offset, next_offset,
""" """
register a vfile register a vfile
""" """
obj = pb.NewObjectInfo(name=str(name), volume_index=int(volume_index), obj = pb.RegisterObjectRequest(name=str(name),
offset=int(offset), volume_index=int(volume_index),
next_offset=int(next_offset), offset=int(offset),
repair_tool=repair_tool) next_offset=int(next_offset),
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/register_object', obj.SerializeToString()) conn.request('POST', '/register_object', obj.SerializeToString())
response = get_rpc_reply(conn, pb.NewObjectReply) response = get_rpc_reply(conn, pb.RegisterObjectReply)
return response return response
def unregister_object(socket_path, name): def unregister_object(socket_path, name, repair_tool=False):
obj = pb.ObjectName(name=str(name)) obj = pb.UnregisterObjectRequest(name=str(name), repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/unregister_object', obj.SerializeToString()) conn.request('POST', '/unregister_object', obj.SerializeToString())
response = get_rpc_reply(conn, pb.DelObjectReply) response = get_rpc_reply(conn, pb.UnregisterObjectReply)
return response return response
def rename_object(socket_path, name, new_name, repair_tool=False): def rename_object(socket_path, name, new_name, repair_tool=False):
rename_info = pb.RenameInfo(name=str(name), new_name=str(new_name), rename_req = pb.RenameObjectRequest(name=str(name), new_name=str(new_name),
repair_tool=repair_tool) repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/rename_object', rename_info.SerializeToString()) conn.request('POST', '/rename_object', rename_req.SerializeToString())
response = get_rpc_reply(conn, pb.RenameReply) response = get_rpc_reply(conn, pb.RenameObjectReply)
return response return response
def quarantine_object(socket_path, name, repair_tool=False): def quarantine_object(socket_path, name, repair_tool=False):
objname = pb.ObjectName(name=str(name), repair_tool=repair_tool) objname = pb.QuarantineObjectRequest(name=str(name),
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/quarantine_object', objname.SerializeToString()) conn.request('POST', '/quarantine_object', objname.SerializeToString())
response = get_rpc_reply(conn, pb.Empty) response = get_rpc_reply(conn, pb.QuarantineObjectReply)
return response return response
def unquarantine_object(socket_path, name, repair_tool=False): def unquarantine_object(socket_path, name, repair_tool=False):
objname = pb.ObjectName(name=str(name), repair_tool=repair_tool) objname = pb.UnquarantineObjectRequest(name=str(name),
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/unquarantine_object', objname.SerializeToString()) conn.request('POST', '/unquarantine_object', objname.SerializeToString())
response = get_rpc_reply(conn, pb.Empty) response = get_rpc_reply(conn, pb.UnquarantineObjectReply)
return response return response
@ -173,12 +179,12 @@ def _list_quarantined_ohashes(socket_path, page_token, page_size):
:param page_size: maximum number of results to be returned :param page_size: maximum number of results to be returned
:return: A list of quarantined object hashes :return: A list of quarantined object hashes
""" """
req_args = pb.ListQuarantinedOHashesInfo(page_token=str(page_token), req_args = pb.ListQuarantinedOHashesRequest(page_token=str(page_token),
page_size=page_size) page_size=page_size)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/list_quarantined_ohashes', conn.request('POST', '/list_quarantined_ohashes',
req_args.SerializeToString()) req_args.SerializeToString())
response = get_rpc_reply(conn, pb.QuarantinedObjectNames) response = get_rpc_reply(conn, pb.ListQuarantinedOHashesReply)
return response return response
@ -195,7 +201,7 @@ def list_quarantined_ohashes(socket_path, page_size=10000):
response = _list_quarantined_ohashes(socket_path, page_token, response = _list_quarantined_ohashes(socket_path, page_token,
page_size) page_size)
for r in response.objects: for r in response.objects:
yield(r) yield (r)
page_token = response.next_page_token page_token = response.next_page_token
if not page_token: if not page_token:
break break
@ -215,13 +221,15 @@ def _list_objects_by_volume(socket_path, volume_index, quarantined, page_token,
:param repair_tool: set to true if caller is a repair tool :param repair_tool: set to true if caller is a repair tool
:return: A list of objects for the volume :return: A list of objects for the volume
""" """
req_args = pb.VolumeIndex(index=volume_index, quarantined=quarantined, req_args = pb.LoadObjectsByVolumeRequest(index=volume_index,
page_token=page_token, page_size=page_size, quarantined=quarantined,
repair_tool=repair_tool) page_token=page_token,
page_size=page_size,
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/load_objects_by_volume', conn.request('POST', '/load_objects_by_volume',
req_args.SerializeToString()) req_args.SerializeToString())
response = get_rpc_reply(conn, pb.LoadObjectsResponse) response = get_rpc_reply(conn, pb.LoadObjectsByVolumeReply)
return response return response
@ -233,7 +241,7 @@ def list_objects_by_volume(socket_path, volume_index, quarantined=False,
quarantined, page_token, page_size, quarantined, page_token, page_size,
repair_tool) repair_tool)
for r in response.objects: for r in response.objects:
yield(r) yield (r)
page_token = response.next_page_token page_token = response.next_page_token
if not page_token: if not page_token:
break break
@ -241,10 +249,11 @@ def list_objects_by_volume(socket_path, volume_index, quarantined=False,
def list_quarantined_ohash(socket_path, prefix, repair_tool=False): def list_quarantined_ohash(socket_path, prefix, repair_tool=False):
len_prefix = len(prefix) len_prefix = len(prefix)
prefix = pb.ObjectPrefix(prefix=str(prefix), repair_tool=repair_tool) prefix = pb.ListQuarantinedOHashRequest(prefix=str(prefix),
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/list_quarantined_ohash', prefix.SerializeToString()) conn.request('POST', '/list_quarantined_ohash', prefix.SerializeToString())
response = get_rpc_reply(conn, pb.LoadObjectsResponse) response = get_rpc_reply(conn, pb.ListQuarantinedOHashReply)
# Caller expects object names without the prefix, similar # Caller expects object names without the prefix, similar
# to os.listdir, not actual objects. # to os.listdir, not actual objects.
@ -259,10 +268,11 @@ def list_quarantined_ohash(socket_path, prefix, repair_tool=False):
def list_prefix(socket_path, prefix, repair_tool=False): def list_prefix(socket_path, prefix, repair_tool=False):
len_prefix = len(prefix) len_prefix = len(prefix)
prefix = str(prefix) prefix = str(prefix)
prefix = pb.ObjectPrefix(prefix=prefix, repair_tool=repair_tool) prefix = pb.LoadObjectsByPrefixRequest(prefix=prefix,
repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/load_objects_by_prefix', prefix.SerializeToString()) conn.request('POST', '/load_objects_by_prefix', prefix.SerializeToString())
response = get_rpc_reply(conn, pb.LoadObjectsResponse) response = get_rpc_reply(conn, pb.LoadObjectsByPrefixReply)
# response.objets is an iterable # response.objets is an iterable
# TBD, caller expects object names without the prefix, similar # TBD, caller expects object names without the prefix, similar
# to os.listdir, not actual objects. # to os.listdir, not actual objects.
@ -279,65 +289,57 @@ def get_object(socket_path, name, is_quarantined=False, repair_tool=False):
""" """
returns an object given its whole key returns an object given its whole key
""" """
object_name = pb.LoadObjectInfo(name=str(name), object_name = pb.LoadObjectRequest(name=str(name),
is_quarantined=is_quarantined, is_quarantined=is_quarantined,
repair_tool=repair_tool) repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/load_object', object_name.SerializeToString()) conn.request('POST', '/load_object', object_name.SerializeToString())
response = get_rpc_reply(conn, pb.Object) response = get_rpc_reply(conn, pb.LoadObjectReply)
return response return response
def list_partitions(socket_path, partition_bits): def list_partitions(socket_path, partition_bits):
list_partitions_info = pb.ListPartitionsInfo( list_partitions_req = pb.ListPartitionsRequest(
partition_bits=partition_bits) partition_bits=partition_bits)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/list_partitions', conn.request('POST', '/list_partitions',
list_partitions_info.SerializeToString()) list_partitions_req.SerializeToString())
response = get_rpc_reply(conn, pb.DirEntries) response = get_rpc_reply(conn, pb.DirEntries)
return response.entry return response.entry
def list_partition(socket_path, partition, partition_bits): def list_partition(socket_path, partition, partition_bits):
list_partition_info = pb.ListPartitionInfo(partition=partition, list_partition_req = pb.ListPartitionRequest(partition=partition,
partition_bits=partition_bits) partition_bits=partition_bits)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/list_partition', conn.request('POST', '/list_partition',
list_partition_info.SerializeToString()) list_partition_req.SerializeToString())
response = get_rpc_reply(conn, pb.DirEntries) response = get_rpc_reply(conn, pb.DirEntries)
return response.entry return response.entry
def list_suffix(socket_path, partition, suffix, partition_bits): def list_suffix(socket_path, partition, suffix, partition_bits):
suffix = str(suffix) suffix = str(suffix)
list_suffix_info = pb.ListSuffixInfo(partition=partition, list_suffix_req = pb.ListSuffixRequest(partition=partition,
suffix=suffix, suffix=suffix,
partition_bits=partition_bits) partition_bits=partition_bits)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/list_suffix', list_suffix_info.SerializeToString()) conn.request('POST', '/list_suffix', list_suffix_req.SerializeToString())
response = get_rpc_reply(conn, pb.DirEntries) response = get_rpc_reply(conn, pb.DirEntries)
return response.entry return response.entry
def list_volumes(socket_path, partition, type, repair_tool=False): def list_volumes(socket_path, partition, type, repair_tool=False):
list_info = pb.ListVolumesInfo(partition=int(partition), type=type, list_req = pb.ListVolumesRequest(partition=int(partition), type=type,
repair_tool=repair_tool) repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/list_volumes', list_info.SerializeToString()) conn.request('POST', '/list_volumes', list_req.SerializeToString())
response = get_rpc_reply(conn, pb.Volumes) response = get_rpc_reply(conn, pb.ListVolumesReply)
return response.volumes return response.volumes
def get_volume_stats(socket_path):
e = pb.Empty()
conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/get_volume_stats', e.SerializeToString())
response = get_rpc_reply(conn, pb.VolumeStats)
return response.volume
def get_volume(socket_path, index, repair_tool=False): def get_volume(socket_path, index, repair_tool=False):
volume_idx = pb.VolumeIndex(index=index, repair_tool=repair_tool) volume_idx = pb.GetVolumeRequest(index=index, repair_tool=repair_tool)
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/get_volume', volume_idx.SerializeToString()) conn.request('POST', '/get_volume', volume_idx.SerializeToString())
response = get_rpc_reply(conn, pb.Volume) response = get_rpc_reply(conn, pb.Volume)
@ -345,16 +347,17 @@ def get_volume(socket_path, index, repair_tool=False):
def get_stats(socket_path): def get_stats(socket_path):
stats_info = pb.GetStatsInfo() stats_req = pb.GetStatsInfo()
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/get_stats', stats_info.SerializeToString()) conn.request('POST', '/get_stats', stats_req.SerializeToString())
response = get_rpc_reply(conn, pb.LoadObjectsResponse) response = get_rpc_reply(conn, pb.GetStatsReply)
return response return response
def get_kv_state(socket_path): def get_kv_state(socket_path):
pb_out = pb.GetKvStateRequest()
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
conn.request('POST', '/get_kv_state') conn.request('POST', '/get_kv_state', pb_out.SerializeToString())
response = get_rpc_reply(conn, pb.KvState) response = get_rpc_reply(conn, pb.KvState)
return response return response
@ -363,5 +366,5 @@ def set_kv_state(socket_path, isClean):
conn = UnixHTTPConnection(socket_path) conn = UnixHTTPConnection(socket_path)
newKvState = pb.KvState(isClean=isClean) newKvState = pb.KvState(isClean=isClean)
conn.request('POST', '/set_kv_state', newKvState.SerializeToString()) conn.request('POST', '/set_kv_state', newKvState.SerializeToString())
response = get_rpc_reply(conn, pb.Empty) response = get_rpc_reply(conn, pb.SetKvStateReply)
return response return response

4
test/unit/obj/test_rpc_http.py
View File

@ -32,8 +32,8 @@ class TestRpcHttp(unittest.TestCase):
with mock.patch("swift.obj.rpc_http.UnixHTTPConnection", with mock.patch("swift.obj.rpc_http.UnixHTTPConnection",
return_value=m_conn): return_value=m_conn):
rpc_http.list_partitions(self.socket_path, self.part_power) rpc_http.list_partitions(self.socket_path, self.part_power)
arg = fmgr_pb2.ListPartitionsInfo(partition_bits=self.part_power) arg = fmgr_pb2.ListPartitionsRequest(
# m_conn[self.socket_path].stub.ListPartitions.assert_called_once_with(arg) partition_bits=self.part_power)
serialized_arg = arg.SerializeToString() serialized_arg = arg.SerializeToString()
m_conn.request.assert_called_once_with('POST', '/list_partitions', m_conn.request.assert_called_once_with('POST', '/list_partitions',
serialized_arg) serialized_arg)

4
test/unit/obj/test_vfile.py
View File

@ -945,7 +945,7 @@ class TestVFileWriter(unittest.TestCase):
conf = {} conf = {}
logger = None logger = None
rpc_reply = fmgr_pb2.Volumes() rpc_reply = fmgr_pb2.ListVolumesReply()
volume = fmgr_pb2.Volume(volume_index=1, volume = fmgr_pb2.Volume(volume_index=1,
volume_type=fmgr_pb2.VOLUME_DEFAULT, volume_type=fmgr_pb2.VOLUME_DEFAULT,
volume_state=fmgr_pb2.STATE_COMPACTION_TARGET, volume_state=fmgr_pb2.STATE_COMPACTION_TARGET,
@ -1059,7 +1059,7 @@ class TestVFileWriter(unittest.TestCase):
for t in test_sets: for t in test_sets:
# build the RPC reply # build the RPC reply
rpc_reply = fmgr_pb2.Volumes() rpc_reply = fmgr_pb2.ListVolumesReply()
for vol in t["volumes"]: for vol in t["volumes"]:
volume = fmgr_pb2.Volume(volume_index=vol["index"], volume = fmgr_pb2.Volume(volume_index=vol["index"],
volume_type=vol["type"], volume_type=vol["type"],