summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2015-02-03 09:49:46 +0000
committerGerrit Code Review <review@openstack.org>2015-02-03 09:49:46 +0000
commit0e2de71ea39831ecf6201bef0290d0d6295a4a6e (patch)
treeda8eceece14cf56f1f3e336d62c319f3498b0b34
parent03fcf5eb481e1f9abcb0216928ef35c91f8ef4f0 (diff)
parent49adcd9cfd91b85489119799e579335ee4730ab0 (diff)
Merge "Changes directory representation"2015.1.0b2
-rw-r--r--hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java4
-rw-r--r--hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java36
-rw-r--r--hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java48
-rw-r--r--hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java234
-rw-r--r--hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java2
-rw-r--r--hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java23
-rw-r--r--hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java6
-rw-r--r--hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java61
8 files changed, 287 insertions, 127 deletions
diff --git a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java
index e32a984..75df4ec 100644
--- a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java
+++ b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java
@@ -899,6 +899,7 @@ public final class SwiftRestClient {
899 * Find objects in a directory 899 * Find objects in a directory
900 * 900 *
901 * @param path path prefix 901 * @param path path prefix
902 * @param addTrailingSlash should a trailing slash be added if there isn't one
902 * @param requestHeaders optional request headers 903 * @param requestHeaders optional request headers
903 * @return byte[] file data or null if the object was not found 904 * @return byte[] file data or null if the object was not found
904 * @throws IOException on IO Faults 905 * @throws IOException on IO Faults
@@ -907,6 +908,7 @@ public final class SwiftRestClient {
907 */ 908 */
908 public byte[] listDeepObjectsInDirectory(SwiftObjectPath path, 909 public byte[] listDeepObjectsInDirectory(SwiftObjectPath path,
909 boolean listDeep, 910 boolean listDeep,
911 boolean addTrailingSlash,
910 final Header... requestHeaders) 912 final Header... requestHeaders)
911 throws IOException { 913 throws IOException {
912 preRemoteCommand("listDeepObjectsInDirectory"); 914 preRemoteCommand("listDeepObjectsInDirectory");
@@ -918,7 +920,7 @@ public final class SwiftRestClient {
918 if (object.startsWith("/")) { 920 if (object.startsWith("/")) {
919 object = object.substring(1); 921 object = object.substring(1);
920 } 922 }
921 if (!object.endsWith("/")) { 923 if (addTrailingSlash && !object.endsWith("/")) {
922 object = object.concat("/"); 924 object = object.concat("/");
923 } 925 }
924 926
diff --git a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
index d876ab5..a96d0fe 100644
--- a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
+++ b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
@@ -21,13 +21,16 @@ package org.apache.hadoop.fs.swift.snative;
21import org.apache.hadoop.fs.FileStatus; 21import org.apache.hadoop.fs.FileStatus;
22import org.apache.hadoop.fs.Path; 22import org.apache.hadoop.fs.Path;
23import org.apache.hadoop.fs.permission.FsPermission; 23import org.apache.hadoop.fs.permission.FsPermission;
24import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
24 25
25/** 26/**
26 * A subclass of {@link FileStatus} that contains the 27 * A subclass of {@link FileStatus} that contains the
27 * Swift-specific rules of when a file is considered to be a directory. 28 * Swift-specific meta-data (e.g. DLO)
28 */ 29 */
29public class SwiftFileStatus extends FileStatus { 30public class SwiftFileStatus extends FileStatus {
30 31
32 private SwiftObjectPath dloPrefix = null;
33
31 public SwiftFileStatus() { 34 public SwiftFileStatus() {
32 } 35 }
33 36
@@ -35,7 +38,17 @@ public class SwiftFileStatus extends FileStatus {
35 boolean isdir, 38 boolean isdir,
36 int block_replication, 39 int block_replication,
37 long blocksize, long modification_time, Path path) { 40 long blocksize, long modification_time, Path path) {
41 this(length, isdir, block_replication, blocksize, modification_time,
42 path, null);
43 }
44
45 public SwiftFileStatus(long length,
46 boolean isdir,
47 int block_replication,
48 long blocksize, long modification_time, Path path,
49 SwiftObjectPath dloPrefix) {
38 super(length, isdir, block_replication, blocksize, modification_time, path); 50 super(length, isdir, block_replication, blocksize, modification_time, path);
51 this.dloPrefix = dloPrefix;
39 } 52 }
40 53
41 public SwiftFileStatus(long length, 54 public SwiftFileStatus(long length,
@@ -51,17 +64,6 @@ public class SwiftFileStatus extends FileStatus {
51 } 64 }
52 65
53 /** 66 /**
54 * Declare that the path represents a directory, which in the
55 * SwiftNativeFileSystem means "is a directory or a 0 byte file"
56 *
57 * @return true if the status is considered to be a file
58 */
59 @Override
60 public boolean isDir() {
61 return super.isDir() || getLen() == 0;
62 }
63
64 /**
65 * A entry is a file if it is not a directory. 67 * A entry is a file if it is not a directory.
66 * By implementing it <i>and not marking as an override</i> this 68 * By implementing it <i>and not marking as an override</i> this
67 * subclass builds and runs in both Hadoop versions. 69 * subclass builds and runs in both Hadoop versions.
@@ -79,13 +81,21 @@ public class SwiftFileStatus extends FileStatus {
79 return isDir(); 81 return isDir();
80 } 82 }
81 83
84 public boolean isDLO() {
85 return dloPrefix != null;
86 }
87
88 public SwiftObjectPath getDLOPrefix() {
89 return dloPrefix;
90 }
91
82 @Override 92 @Override
83 public String toString() { 93 public String toString() {
84 StringBuilder sb = new StringBuilder(); 94 StringBuilder sb = new StringBuilder();
85 sb.append(getClass().getSimpleName()); 95 sb.append(getClass().getSimpleName());
86 sb.append("{ "); 96 sb.append("{ ");
87 sb.append("path=").append(getPath()); 97 sb.append("path=").append(getPath());
88 sb.append("; isDirectory=").append(isDir()); 98 sb.append("; isDirectory=").append(isDirectory());
89 sb.append("; length=").append(getLen()); 99 sb.append("; length=").append(getLen());
90 sb.append("; blocksize=").append(getBlockSize()); 100 sb.append("; blocksize=").append(getBlockSize());
91 sb.append("; modification_time=").append(getModificationTime()); 101 sb.append("; modification_time=").append(getModificationTime());
diff --git a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
index 016a35c..c172d9f 100644
--- a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
+++ b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
@@ -200,27 +200,6 @@ public class SwiftNativeFileSystem extends FileSystem {
200 return store.getBlocksize(); 200 return store.getBlocksize();
201 } 201 }
202 202
203 @Override
204 public boolean isFile(Path f) throws IOException {
205 try {
206 FileStatus fileStatus = getFileStatus(f);
207 return !SwiftUtils.isDirectory(fileStatus);
208 } catch (FileNotFoundException e) {
209 return false; // f does not exist
210 }
211 }
212
213 @Override
214 public boolean isDirectory(Path f) throws IOException {
215
216 try {
217 FileStatus fileStatus = getFileStatus(f);
218 return SwiftUtils.isDirectory(fileStatus);
219 } catch (FileNotFoundException e) {
220 return false; // f does not exist
221 }
222 }
223
224 /** 203 /**
225 * Return an array containing hostnames, offset and size of 204 * Return an array containing hostnames, offset and size of
226 * portions of the given file. For a nonexistent 205 * portions of the given file. For a nonexistent
@@ -239,6 +218,9 @@ public class SwiftNativeFileSystem extends FileSystem {
239 if (file == null) { 218 if (file == null) {
240 return null; 219 return null;
241 } 220 }
221 if (file.isDir()) {
222 return new BlockLocation[0];
223 }
242 224
243 if (start < 0 || len < 0) { 225 if (start < 0 || len < 0) {
244 throw new IllegalArgumentException("Negative start or len parameter" + 226 throw new IllegalArgumentException("Negative start or len parameter" +
@@ -251,11 +233,15 @@ public class SwiftNativeFileSystem extends FileSystem {
251 // Check if requested file in Swift is more than 5Gb. In this case 233 // Check if requested file in Swift is more than 5Gb. In this case
252 // each block has its own location -which may be determinable 234 // each block has its own location -which may be determinable
253 // from the Swift client API, depending on the remote server 235 // from the Swift client API, depending on the remote server
254 final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath(), 236 final FileStatus[] listOfFileBlocks;
255 false, 237 if (file instanceof SwiftFileStatus && ((SwiftFileStatus)file).isDLO()) {
256 true); 238 listOfFileBlocks = store.listSegments((SwiftFileStatus)file, true);
239 } else {
240 listOfFileBlocks = null;
241 }
242
257 List<URI> locations = new ArrayList<URI>(); 243 List<URI> locations = new ArrayList<URI>();
258 if (listOfFileBlocks.length > 1) { 244 if (listOfFileBlocks != null && listOfFileBlocks.length > 1) {
259 for (FileStatus fileStatus : listOfFileBlocks) { 245 for (FileStatus fileStatus : listOfFileBlocks) {
260 if (SwiftObjectPath.fromPath(uri, fileStatus.getPath()) 246 if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
261 .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) { 247 .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
@@ -386,7 +372,7 @@ public class SwiftNativeFileSystem extends FileSystem {
386 //find out about the path 372 //find out about the path
387 fileStatus = getFileStatus(directory); 373 fileStatus = getFileStatus(directory);
388 374
389 if (!SwiftUtils.isDirectory(fileStatus)) { 375 if (!fileStatus.isDir()) {
390 //if it's a file, raise an error 376 //if it's a file, raise an error
391 throw new SwiftNotDirectoryException(directory, 377 throw new SwiftNotDirectoryException(directory,
392 String.format(": can't mkdir since it exists and is not a directory: %s", 378 String.format(": can't mkdir since it exists and is not a directory: %s",
@@ -433,7 +419,15 @@ public class SwiftNativeFileSystem extends FileSystem {
433 if (LOG.isDebugEnabled()) { 419 if (LOG.isDebugEnabled()) {
434 LOG.debug("SwiftFileSystem.listStatus for: " + path); 420 LOG.debug("SwiftFileSystem.listStatus for: " + path);
435 } 421 }
436 return store.listSubPaths(makeAbsolute(path), false, true); 422 Path absolutePath = makeAbsolute(path);
423 try {
424 return store.listSubPaths(absolutePath, false, true);
425 } catch (FileNotFoundException e) {
426 /* path is not directory. try to get file status */
427 return new FileStatus[] {
428 getFileStatus(absolutePath)
429 };
430 }
437 } 431 }
438 432
439 /** 433 /**
diff --git a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
index d92a065..82f2d84 100644
--- a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
+++ b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
@@ -210,15 +210,38 @@ public class SwiftNativeFileSystemStore {
210 throws IOException, FileNotFoundException { 210 throws IOException, FileNotFoundException {
211 211
212 SwiftObjectPath objectPath = toObjectPath(path); 212 SwiftObjectPath objectPath = toObjectPath(path);
213 final Header[] headers = stat(objectPath, newest); 213
214 //no headers is treated as a missing file 214 // remove trailing slash because FileStatus must not include that
215 if (headers.length == 0) { 215 Path statusPath = path;
216 if (statusPath.toUri().toString().endsWith("/")) {
217 String pathUri = statusPath.toUri().toString();
218 if (pathUri.length() > 1)
219 statusPath = new Path(pathUri.substring(0, pathUri.length() - 1));
220 }
221
222 Header[] headers = null;
223 try {
224 headers = stat(objectPath, newest);
225 } catch (FileNotFoundException e) {
226 // if path is pseudo-directory, ignore FileNotFoundException.
227 }
228 //no headers is treated as a missing file or pseudo-directory
229 if (headers == null || headers.length == 0) {
230 if (existsPseudoDirectory(objectPath)) {
231 return new SwiftFileStatus(0,
232 true,
233 1,
234 getBlocksize(),
235 System.currentTimeMillis(),
236 getCorrectSwiftPath(statusPath));
237 }
216 throw new FileNotFoundException("Not Found " + path.toUri()); 238 throw new FileNotFoundException("Not Found " + path.toUri());
217 } 239 }
218 240
219 boolean isDir = false; 241 boolean isDir = false;
220 long length = 0; 242 long length = 0;
221 long lastModified = 0 ; 243 long lastModified = 0 ;
244 SwiftObjectPath dloPrefix = null;
222 for (Header header : headers) { 245 for (Header header : headers) {
223 String headerName = header.getName(); 246 String headerName = header.getName();
224 if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) || 247 if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) ||
@@ -237,18 +260,28 @@ public class SwiftNativeFileSystemStore {
237 throw new SwiftException("Failed to parse " + header.toString(), e); 260 throw new SwiftException("Failed to parse " + header.toString(), e);
238 } 261 }
239 } 262 }
263 if (headerName.equals(SwiftProtocolConstants.X_OBJECT_MANIFEST)) {
264 String[] values = header.getValue().split("/", 2);
265 if (values.length == 2) {
266 dloPrefix = new SwiftObjectPath(values[0], "/" + values[1]);
267 }
268 }
240 } 269 }
241 if (lastModified == 0) { 270 if (lastModified == 0) {
242 lastModified = System.currentTimeMillis(); 271 lastModified = System.currentTimeMillis();
243 } 272 }
273 if (objectPath.toString().endsWith("/")) {
274 isDir = true;
275 }
244 276
245 Path correctSwiftPath = getCorrectSwiftPath(path); 277 Path correctSwiftPath = getCorrectSwiftPath(statusPath);
246 return new SwiftFileStatus(length, 278 return new SwiftFileStatus(length,
247 isDir, 279 isDir,
248 1, 280 1,
249 getBlocksize(), 281 getBlocksize(),
250 lastModified, 282 lastModified,
251 correctSwiftPath); 283 correctSwiftPath,
284 dloPrefix);
252 } 285 }
253 286
254 private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws 287 private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
@@ -264,6 +297,39 @@ public class SwiftNativeFileSystemStore {
264 return headers; 297 return headers;
265 } 298 }
266 299
300 private boolean existsPseudoDirectory(SwiftObjectPath path) {
301 try {
302 String pseudoDirName = path.getObject();
303 if (pseudoDirName.endsWith("/")) {
304 String obj = path.getObject();
305 path = new SwiftObjectPath(path.getContainer(),
306 obj.substring(0, obj.length() - 1));
307 } else {
308 pseudoDirName = pseudoDirName.concat("/");
309 }
310
311 final byte[] bytes;
312 bytes = swiftRestClient.listDeepObjectsInDirectory(path, false, false);
313
314 final CollectionType collectionType = JSONUtil.getJsonMapper().
315 getTypeFactory().constructCollectionType(List.class,
316 SwiftObjectFileStatus.class);
317
318 final List<SwiftObjectFileStatus> fileStatusList =
319 JSONUtil.toObject(new String(bytes), collectionType);
320
321 for (SwiftObjectFileStatus status : fileStatusList) {
322 if (pseudoDirName.equals(status.getSubdir())) {
323 return true;
324 }
325 }
326 return false;
327
328 } catch (Exception e) {
329 return false;
330 }
331 }
332
267 /** 333 /**
268 * Get the object as an input stream 334 * Get the object as an input stream
269 * 335 *
@@ -401,18 +467,22 @@ public class SwiftNativeFileSystemStore {
401 * @param path working path 467 * @param path working path
402 * @param listDeep ask for all the data 468 * @param listDeep ask for all the data
403 * @param newest ask for the newest data 469 * @param newest ask for the newest data
470 * @param addTrailingSlash should a trailing slash be added if there isn't one
404 * @return Collection of file statuses 471 * @return Collection of file statuses
405 * @throws IOException IO problems 472 * @throws IOException IO problems
406 * @throws FileNotFoundException if the path does not exist 473 * @throws FileNotFoundException if the path does not exist
407 */ 474 */
408 private List<FileStatus> listDirectory(SwiftObjectPath path, 475 private List<FileStatus> listDirectory(SwiftObjectPath path,
409 boolean listDeep, 476 boolean listDeep,
410 boolean newest) throws IOException { 477 boolean newest,
478 boolean addTrailingSlash)
479 throws IOException {
411 final byte[] bytes; 480 final byte[] bytes;
412 final ArrayList<FileStatus> files = new ArrayList<FileStatus>(); 481 final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
413 final Path correctSwiftPath = getCorrectSwiftPath(path); 482 final Path correctSwiftPath = getCorrectSwiftPath(path);
414 try { 483 try {
415 bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep); 484 bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep,
485 addTrailingSlash);
416 } catch (FileNotFoundException e) { 486 } catch (FileNotFoundException e) {
417 if (LOG.isDebugEnabled()) { 487 if (LOG.isDebugEnabled()) {
418 LOG.debug("" + 488 LOG.debug("" +
@@ -468,14 +538,25 @@ public class SwiftNativeFileSystemStore {
468 return files; 538 return files;
469 } 539 }
470 540
541 String pathWithSlash = path.getObject();
542 if (!pathWithSlash.endsWith("/")) {
543 pathWithSlash = pathWithSlash.concat("/");
544 }
471 for (SwiftObjectFileStatus status : fileStatusList) { 545 for (SwiftObjectFileStatus status : fileStatusList) {
472 if (status.getName() != null) { 546 String name = status.getName();
473 files.add(new SwiftFileStatus(status.getBytes(), 547 if (name == null) {
474 status.getBytes() == 0, 548 name = status.getSubdir();
475 1, 549 }
476 getBlocksize(), 550 if (name == null || name.equals(pathWithSlash)) {
477 status.getLast_modified().getTime(), 551 continue;
478 getCorrectSwiftPath(new Path(status.getName())))); 552 }
553
554 if (!name.endsWith("/")) {
555 final Path filePath = getCorrectSwiftPath(new Path(name));
556 files.add(getObjectMetadata(filePath, newest));
557 } else {
558 final Path dirPath = getCorrectSwiftPath(toDirPath(new Path(name)));
559 files.add(getObjectMetadata(dirPath, newest));
479 } 560 }
480 } 561 }
481 562
@@ -498,7 +579,7 @@ public class SwiftNativeFileSystemStore {
498 boolean recursive, 579 boolean recursive,
499 boolean newest) throws IOException { 580 boolean newest) throws IOException {
500 final Collection<FileStatus> fileStatuses; 581 final Collection<FileStatus> fileStatuses;
501 fileStatuses = listDirectory(toDirPath(path), recursive, newest); 582 fileStatuses = listDirectory(toDirPath(path), recursive, newest, true);
502 return fileStatuses.toArray(new FileStatus[fileStatuses.size()]); 583 return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
503 } 584 }
504 585
@@ -527,7 +608,7 @@ public class SwiftNativeFileSystemStore {
527 608
528 private SwiftObjectPath toDirPath(Path path) throws 609 private SwiftObjectPath toDirPath(Path path) throws
529 SwiftConfigurationException { 610 SwiftConfigurationException {
530 return SwiftObjectPath.fromPath(uri, path, false); 611 return SwiftObjectPath.fromPath(uri, path, true);
531 } 612 }
532 613
533 private SwiftObjectPath toObjectPath(Path path) throws 614 private SwiftObjectPath toObjectPath(Path path) throws
@@ -554,12 +635,17 @@ public class SwiftNativeFileSystemStore {
554 /** 635 /**
555 * deletes object from Swift 636 * deletes object from Swift
556 * 637 *
557 * @param path path to delete 638 * @param status FileStatus to delete
558 * @return true if the path was deleted by this specific operation. 639 * @return true if the path was deleted by this specific operation.
559 * @throws IOException on a failure 640 * @throws IOException on a failure
560 */ 641 */
561 public boolean deleteObject(Path path) throws IOException { 642 public boolean deleteObject(FileStatus status) throws IOException {
562 SwiftObjectPath swiftObjectPath = toObjectPath(path); 643 SwiftObjectPath swiftObjectPath;
644 if (status.isDir()) {
645 swiftObjectPath = toDirPath(status.getPath());
646 } else {
647 swiftObjectPath = toObjectPath(status.getPath());
648 }
563 if (!SwiftUtils.isRootDir(swiftObjectPath)) { 649 if (!SwiftUtils.isRootDir(swiftObjectPath)) {
564 return swiftRestClient.delete(swiftObjectPath); 650 return swiftRestClient.delete(swiftObjectPath);
565 } else { 651 } else {
@@ -571,18 +657,6 @@ public class SwiftNativeFileSystemStore {
571 } 657 }
572 658
573 /** 659 /**
574 * deletes a directory from Swift. This is not recursive
575 *
576 * @param path path to delete
577 * @return true if the path was deleted by this specific operation -or
578 * the path was root and not acted on.
579 * @throws IOException on a failure
580 */
581 public boolean rmdir(Path path) throws IOException {
582 return deleteObject(path);
583 }
584
585 /**
586 * Does the object exist 660 * Does the object exist
587 * 661 *
588 * @param path object path 662 * @param path object path
@@ -635,22 +709,32 @@ public class SwiftNativeFileSystemStore {
635 } 709 }
636 boolean renamingOnToSelf = src.equals(dst); 710 boolean renamingOnToSelf = src.equals(dst);
637 711
638 SwiftObjectPath srcObject = toObjectPath(src); 712 final SwiftFileStatus srcMetadata;
639 SwiftObjectPath destObject = toObjectPath(dst); 713 srcMetadata = getObjectMetadata(src);
640 714 SwiftObjectPath srcObject;
715 if (srcMetadata.isDirectory()) {
716 srcObject = toDirPath(src);
717 } else {
718 srcObject = toObjectPath(src);
719 }
641 if (SwiftUtils.isRootDir(srcObject)) { 720 if (SwiftUtils.isRootDir(srcObject)) {
642 throw new SwiftOperationFailedException("cannot rename root dir"); 721 throw new SwiftOperationFailedException("cannot rename root dir");
643 } 722 }
644 723
645 final SwiftFileStatus srcMetadata;
646 srcMetadata = getObjectMetadata(src);
647 SwiftFileStatus dstMetadata; 724 SwiftFileStatus dstMetadata;
725 SwiftObjectPath destObject;
648 try { 726 try {
649 dstMetadata = getObjectMetadata(dst); 727 dstMetadata = getObjectMetadata(dst);
728 if (dstMetadata.isDirectory()) {
729 destObject = toDirPath(dst);
730 } else {
731 destObject = toObjectPath(dst);
732 }
650 } catch (FileNotFoundException e) { 733 } catch (FileNotFoundException e) {
651 //destination does not exist. 734 //destination does not exist.
652 LOG.debug("Destination does not exist"); 735 LOG.debug("Destination does not exist");
653 dstMetadata = null; 736 dstMetadata = null;
737 destObject = toObjectPath(dst);
654 } 738 }
655 739
656 //check to see if the destination parent directory exists 740 //check to see if the destination parent directory exists
@@ -670,13 +754,13 @@ public class SwiftNativeFileSystemStore {
670 } 754 }
671 755
672 boolean destExists = dstMetadata != null; 756 boolean destExists = dstMetadata != null;
673 boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata); 757 boolean destIsDir = destExists && dstMetadata.isDirectory();
674 //calculate the destination 758 //calculate the destination
675 SwiftObjectPath destPath; 759 SwiftObjectPath destPath;
676 760
677 //enum the child entries and everything underneath 761 //enum the child entries and everything underneath
678 List<FileStatus> childStats = listDirectory(srcObject, true, true); 762 List<FileStatus> childStats = listDirectory(srcObject, true, true, true);
679 boolean srcIsFile = !srcMetadata.isDir(); 763 boolean srcIsFile = !srcMetadata.isDirectory();
680 if (srcIsFile) { 764 if (srcIsFile) {
681 765
682 //source is a simple file OR a partitioned file 766 //source is a simple file OR a partitioned file
@@ -718,7 +802,7 @@ public class SwiftNativeFileSystemStore {
718 copyObject(srcObject, destPath); 802 copyObject(srcObject, destPath);
719 for (FileStatus stat : childStats) { 803 for (FileStatus stat : childStats) {
720 SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat); 804 SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
721 deleteObject(stat.getPath()); 805 deleteObject(stat);
722 } 806 }
723 807
724 swiftRestClient.delete(srcObject); 808 swiftRestClient.delete(srcObject);
@@ -746,7 +830,7 @@ public class SwiftNativeFileSystemStore {
746 // #3 destination doesn't exist: create a new dir with that name 830 // #3 destination doesn't exist: create a new dir with that name
747 targetPath = dst; 831 targetPath = dst;
748 } 832 }
749 SwiftObjectPath targetObjectPath = toObjectPath(targetPath); 833 SwiftObjectPath targetObjectPath = toDirPath(targetPath);
750 //final check for any recursive operations 834 //final check for any recursive operations
751 if (srcObject.isEqualToOrParentOf(targetObjectPath)) { 835 if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
752 //you can't rename a directory onto itself 836 //you can't rename a directory onto itself
@@ -780,11 +864,19 @@ public class SwiftNativeFileSystemStore {
780 + "; copyDestSubPath=" + copyDestSubPath 864 + "; copyDestSubPath=" + copyDestSubPath
781 + "; copyDestPath=" + copyDestPath); 865 + "; copyDestPath=" + copyDestPath);
782 } 866 }
783 SwiftObjectPath copyDestination = toObjectPath(copyDestPath); 867 SwiftObjectPath copySource;
868 SwiftObjectPath copyDestination;
869
870 if (fileStatus.isDir()) {
871 copySource = toDirPath(copySourcePath);
872 copyDestination = toDirPath(copyDestPath);
873 } else {
874 copySource = toObjectPath(copySourcePath);
875 copyDestination = toObjectPath(copyDestPath);
876 }
784 877
785 try { 878 try {
786 copyThenDeleteObject(toObjectPath(copySourcePath), 879 copyThenDeleteObject(copySource, copyDestination);
787 copyDestination);
788 } catch (FileNotFoundException e) { 880 } catch (FileNotFoundException e) {
789 LOG.info("Skipping rename of " + copySourcePath); 881 LOG.info("Skipping rename of " + copySourcePath);
790 } 882 }
@@ -1008,14 +1100,12 @@ public class SwiftNativeFileSystemStore {
1008 //don't mind if the directory has changed 1100 //don't mind if the directory has changed
1009 //list all entries under this directory. 1101 //list all entries under this directory.
1010 //this will throw FileNotFoundException if the file isn't there 1102 //this will throw FileNotFoundException if the file isn't there
1011 FileStatus[] statuses = listSubPaths(absolutePath, true, askForNewest); 1103 FileStatus[] statuses;
1012 if (statuses == null) { 1104 try {
1013 //the directory went away during the non-atomic stages of the operation. 1105 statuses = listSubPaths(absolutePath, true, askForNewest);
1014 // Return false as it was not this thread doing the deletion. 1106 } catch (IOException e) {
1015 SwiftUtils.debug(LOG, "Path '%s' has no status -it has 'gone away'", 1107 // absolutePath is nonexistent
1016 absolutePath, 1108 statuses = new FileStatus[0];
1017 recursive);
1018 return false;
1019 } 1109 }
1020 int filecount = statuses.length; 1110 int filecount = statuses.length;
1021 SwiftUtils.debug(LOG, "Path '%s' %d status entries'", 1111 SwiftUtils.debug(LOG, "Path '%s' %d status entries'",
@@ -1024,7 +1114,7 @@ public class SwiftNativeFileSystemStore {
1024 1114
1025 if (filecount == 0) { 1115 if (filecount == 0) {
1026 //it's an empty directory or a path 1116 //it's an empty directory or a path
1027 rmdir(absolutePath); 1117 deleteObject(fileStatus);
1028 return true; 1118 return true;
1029 } 1119 }
1030 1120
@@ -1036,14 +1126,14 @@ public class SwiftNativeFileSystemStore {
1036 // 1 entry => simple file and it is the target 1126 // 1 entry => simple file and it is the target
1037 //simple file: delete it 1127 //simple file: delete it
1038 SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath); 1128 SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath);
1039 deleteObject(absolutePath); 1129 deleteObject(fileStatus);
1040 return true; 1130 return true;
1041 } 1131 }
1042 1132
1043 //>1 entry implies directory with children. Run through them, 1133 //>1 entry implies directory with children. Run through them,
1044 // but first check for the recursive flag and reject it *unless it looks 1134 // but first check for the recursive flag and reject it *unless it looks
1045 // like a partitioned file (len > 0 && has children) 1135 // like a partitioned file (len > 0 && has children)
1046 if (!fileStatus.isDir()) { 1136 if (!fileStatus.isDirectory()) {
1047 LOG.debug("Multiple child entries but entry has data: assume partitioned"); 1137 LOG.debug("Multiple child entries but entry has data: assume partitioned");
1048 } else if (!recursive) { 1138 } else if (!recursive) {
1049 //if there are children, unless this is a recursive operation, fail immediately 1139 //if there are children, unless this is a recursive operation, fail immediately
@@ -1057,7 +1147,7 @@ public class SwiftNativeFileSystemStore {
1057 for (FileStatus entryStatus : statuses) { 1147 for (FileStatus entryStatus : statuses) {
1058 Path entryPath = entryStatus.getPath(); 1148 Path entryPath = entryStatus.getPath();
1059 try { 1149 try {
1060 boolean deleted = deleteObject(entryPath); 1150 boolean deleted = deleteObject(entryStatus);
1061 if (!deleted) { 1151 if (!deleted) {
1062 SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing", 1152 SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing",
1063 entryPath); 1153 entryPath);
@@ -1072,8 +1162,38 @@ public class SwiftNativeFileSystemStore {
1072 } 1162 }
1073 //now delete self 1163 //now delete self
1074 SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath); 1164 SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath);
1075 deleteObject(absolutePath); 1165 deleteObject(fileStatus);
1076 1166
1077 return true; 1167 return true;
1078 } 1168 }
1169
1170 /**
1171 * List all segments in dynamic large object.
1172 *
1173 * @param file SwiftFileStatus of large object
1174 * @param newest ask for the newest, or can some out of date data work?
1175 * @return the file statuses, or an empty array if there are no segments
1176 * @throws IOException on IO problems
1177 */
1178 public FileStatus[] listSegments(SwiftFileStatus file, boolean newest)
1179 throws IOException {
1180
1181 if (file.getDLOPrefix() == null) {
1182 return new FileStatus[0];
1183 }
1184
1185 final List<FileStatus> objects;
1186 objects = listDirectory(file.getDLOPrefix(), true, newest, false);
1187
1188 final ArrayList<FileStatus> segments;
1189 segments = new ArrayList<FileStatus>(objects.size());
1190
1191 for (FileStatus status : objects) {
1192 if (!status.isDir()) {
1193 segments.add(status);
1194 }
1195 }
1196
1197 return segments.toArray(new FileStatus[segments.size()]);
1198 }
1079} 1199}
diff --git a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
index ca8adc6..27d7729 100644
--- a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
+++ b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
@@ -26,7 +26,7 @@ import java.util.Date;
26 * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS. 26 * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
27 */ 27 */
28 28
29class SwiftObjectFileStatus { 29public class SwiftObjectFileStatus {
30 private long bytes; 30 private long bytes;
31 private String content_type; 31 private String content_type;
32 private String hash; 32 private String hash;
diff --git a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
index c743818..887ed99 100644
--- a/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
+++ b/hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
@@ -59,29 +59,6 @@ public final class SwiftUtils {
59 } 59 }
60 60
61 /** 61 /**
62 * This test contains the is-directory logic for Swift, so if
63 * changed there is only one place for it.
64 *
65 * @param fileStatus status to examine
66 * @return true if we consider this status to be representative of a
67 * directory.
68 */
69 public static boolean isDirectory(FileStatus fileStatus) {
70 return fileStatus.isDir() || isFilePretendingToBeDirectory(fileStatus);
71 }
72
73 /**
74 * Test for the entry being a file that is treated as if it is a
75 * directory
76 *
77 * @param fileStatus status
78 * @return true if it meets the rules for being a directory
79 */
80 public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
81 return fileStatus.getLen() == 0;
82 }
83
84 /**
85 * Predicate: Is a swift object referring to the root direcory? 62 * Predicate: Is a swift object referring to the root direcory?
86 * @param swiftObject object to probe 63 * @param swiftObject object to probe
87 * @return true iff the object refers to the root 64 * @return true iff the object refers to the root
diff --git a/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java b/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
index 8ad0ca4..18fa466 100644
--- a/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
+++ b/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
@@ -286,4 +286,10 @@ public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
286 } 286 }
287 } 287 }
288 288
289 @Test(timeout = SWIFT_TEST_TIMEOUT)
290 public void testExistsRoot() throws Exception {
291 Path path = new Path("/");
292 assertTrue("exists('/') returned false", fs.exists(path));
293 }
294
289} 295}
diff --git a/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java b/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
index ae59820..977f0eb 100644
--- a/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
+++ b/hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
@@ -20,11 +20,21 @@ package org.apache.hadoop.fs.swift;
20 20
21import org.apache.hadoop.fs.FileStatus; 21import org.apache.hadoop.fs.FileStatus;
22import org.apache.hadoop.fs.Path; 22import org.apache.hadoop.fs.Path;
23import org.apache.hadoop.fs.swift.http.SwiftRestClient;
23import org.apache.hadoop.fs.swift.snative.SwiftFileStatus; 24import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
25import org.apache.hadoop.fs.swift.snative.SwiftObjectFileStatus;
26import org.apache.hadoop.fs.swift.util.JSONUtil;
27import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
24import org.apache.hadoop.fs.swift.util.SwiftTestUtils; 28import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
29import org.codehaus.jackson.map.type.CollectionType;
25import org.junit.Test; 30import org.junit.Test;
26 31
27import java.io.FileNotFoundException; 32import java.io.FileNotFoundException;
33import java.net.URI;
34import java.util.ArrayList;
35import java.util.List;
36
37import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
28 38
29/** 39/**
30 * Test swift-specific directory logic. 40 * Test swift-specific directory logic.
@@ -35,16 +45,16 @@ public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
35 45
36 /** 46 /**
37 * Asserts that a zero byte file has a status of file and not 47 * Asserts that a zero byte file has a status of file and not
38 * directory or symlink 48 * file or symlink
39 * 49 *
40 * @throws Exception on failures 50 * @throws Exception on failures
41 */ 51 */
42 @Test(timeout = SWIFT_TEST_TIMEOUT) 52 @Test(timeout = SWIFT_TEST_TIMEOUT)
43 public void testZeroByteFilesAreDirectories() throws Exception { 53 public void testZeroByteFilesAreFiles() throws Exception {
44 Path src = path("/test/testZeroByteFilesAreFiles"); 54 Path src = path("/test/testZeroByteFilesAreFiles");
45 //create a zero byte file 55 //create a zero byte file
46 SwiftTestUtils.touch(fs, src); 56 SwiftTestUtils.touch(fs, src);
47 SwiftTestUtils.assertIsDirectory(fs, src); 57 SwiftTestUtils.assertIsFile(fs, src);
48 } 58 }
49 59
50 @Test(timeout = SWIFT_TEST_TIMEOUT) 60 @Test(timeout = SWIFT_TEST_TIMEOUT)
@@ -79,8 +89,8 @@ public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
79 89
80 Path src = path("/test/file"); 90 Path src = path("/test/file");
81 91
82 //create a zero byte file 92 //create a directory
83 SwiftTestUtils.touch(fs, src); 93 fs.mkdirs(src);
84 //stat it 94 //stat it
85 statuses = fs.listStatus(test); 95 statuses = fs.listStatus(test);
86 statusString = statusToString(test.toString(), statuses); 96 statusString = statusToString(test.toString(), statuses);
@@ -138,4 +148,45 @@ public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
138 assertFalse(status.isDir()); 148 assertFalse(status.isDir());
139 } 149 }
140 150
151 /**
152 * Asserts that a mkdir with trailing slash makes single object only
153 *
154 * @throws Exception on failures
155 */
156 @Test(timeout = SWIFT_TEST_TIMEOUT)
157 public void testDirectoriesUseURI() throws Exception {
158 cleanup("testDirectoriesUseURI", fs, "/");
159 Path test = new Path(fs.getUri().resolve("/test/"));
160 mkdirs(test);
161 assertExists("created test directory", test);
162 FileStatus[] statuses = fs.listStatus(test);
163 String statusString = statusToString(test.toString(), statuses);
164 assertEquals("Wrong number of elements in file status " + statusString, 0,
165 statuses.length);
166
167 String[] objects = getRawObjectNames();
168 assertEquals("Wrong number of objects in swift", 1, objects.length);
169 assertEquals("Wrong directory name", "/test/", objects[0]);
170 }
171
172 private String[] getRawObjectNames() throws Exception {
173 SwiftRestClient client;
174 client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf());
175 SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/"));
176 byte[] bytes = client.listDeepObjectsInDirectory(path, true, true);
177 final CollectionType collectionType = JSONUtil.getJsonMapper().
178 getTypeFactory().constructCollectionType(List.class,
179 SwiftObjectFileStatus.class);
180 final List<SwiftObjectFileStatus> fileStatusList =
181 JSONUtil.toObject(new String(bytes), collectionType);
182 final ArrayList<String> objects = new ArrayList();
183 for (SwiftObjectFileStatus status : fileStatusList) {
184 if (status.getName() != null) {
185 objects.add(status.getName());
186 } else if (status.getSubdir() != null) {
187 objects.add(status.getSubdir());
188 }
189 }
190 return objects.toArray(new String[objects.size()]);
191 }
141} 192}