Project rename, plus new approach

This patch changes the project from a series of hooks, to a filesystem
abstraction shim. Instead of trying to modify our JSON directly, we
intercept filesystem actions to create the appropriate folder structure.
This commit is contained in:
Michael Krotscheck 2016-04-08 07:19:25 -07:00
parent 9d92c2f999
commit 9ec3da6e82
No known key found for this signature in database
GPG Key ID: 20E618D878DE38AB
4 changed files with 56 additions and 68 deletions

View File

@ -1,7 +1,10 @@
# registry-static-afs
This project provides a rewrite hook for the registry-static project, allowing the mirrored files to be hosted on AFS.
# sorting-fs-blob-store
AFS has a practical folder size limit of ~64K entries. In order to accomodate as many packages as are in the npm registry, we store them in subfolders, sorted by the first letter of the package name. For example: `/foo` becomes `/f/foo`, `/bar` becomes `/b/bar`, `/q` becomes `/q/q`. File path resolution is handled via apache's `mod_rewrite`.
This project provides a renaming shim on top of fs-blob-store, to permit the storing of large folders on AFS.
AFS has a practical folder size limit of ~64K entries. In order to accomodate folders with more files than this (such as the npm registry), we store them in subfolders, sorted by the first letter of the package name. For example: `/foo` becomes `/f/foo`, `/bar` becomes `/b/bar`, `/q` becomes `/q/q`. Only files that begin with alphanumeric characters are sorted.
Accessing these files via sorting-fs-blob-store is transparent. Accessing them via other methods can be accomplished via rewrite rules (for instance, apache's mod_rewrite).
This module is in use by OpenStack's Infrastructure team, to run the npm portion of our unified mirrors.
@ -14,7 +17,7 @@ This module is in use by OpenStack's Infrastructure team, to run the npm portion
# Run the registry script
registry-static -d my.registry.com -o /var/www/registry \
--hooks registry-static-afs
--blobstore sorting-fs-blob-store
### Some useful development commands

View File

@ -1,66 +1,34 @@
(function() {
'use strict';
/**
* No-op hook, executes with no modifications to the current event.
*
* @param {{}} data The currently modified data.
* @param {Function} cb Asynchronous callback with the signature cb(error, shouldSave);
* @returns {void}
*/
function noop (data, cb) {
cb(null, true);
var options = require('./lib/args');
var blobstore = require('fs-blob-store')(options.dir);
function afsify (opts) {
if (typeof opts === 'string') {
// Packages that start with alphanumerics are rewritten:
// foo -> f/foo, bar -> b/bar
opts = opts.replace(/^\/?([a-zA-Z0-9])/, function(match, submatch) {
return submatch + '/' + submatch;
});
return opts;
}
return opts;
}
module.exports = {
/**
* Called before any data is written, at the beginning of processing a change. If the callback
* is called with an error or false, no more processing will be done for this change, and no
* files will be written.
*/
beforeAll: noop,
/**
* Called after all the data is written, at the end of processing a change. If there is no
* error, the callback parameters are ignored.
*/
afterAll: noop,
/**
* Called before writing an update to the top-level index.json.
*/
globalIndexJson: noop,
/**
* Called before writing a package's main index.json.
*/
indexJson: noop,
/**
* Called before writing the index.json for a particular package version.
*/
versionJson: noop,
/**
* Called before downloading/verifying/writing a package tarball.
*/
tarball: noop,
/**
* Called after downloading/verifying/writing a package tarball. If there is no error, the
* callback parameters are ignored.
*/
afterTarball: noop,
/**
* Called before doing anything else at start time.
*/
startup: noop,
/**
* Called in order to check the sha1sum of a tarball. Calling back with true implies the shasum
* passed. Default is in lib/defaultShasumCheck.js.
*/
shasumCheck: noop
createWriteStream: function(opts, cb) {
return blobstore.createWriteStream(afsify(opts), cb);
},
createReadStream: function(key, opts) {
return blobstore.createReadStream(afsify(key), opts);
},
exists: function(opts, cb) {
return blobstore.exists(afsify(opts), cb);
},
remove: function(opts, cb) {
return blobstore.remove(afsify(opts), cb);
}
};
})();

10
lib/args.js Normal file
View File

@ -0,0 +1,10 @@
(function() {
'use strict';
var yargs = require('yargs');
var options = yargs(process.argv);
options.alias('dir', 'o');
module.exports = options.argv;
})();

View File

@ -1,23 +1,30 @@
{
"license": "Apache-2.0",
"name": "registry-static-afs",
"name": "sorting-fs-blob-store",
"version": "1.0.0",
"description": "A module for the registry-static mirror package, hosting files in a folder structure that works around AFS's folder size limit.",
"description": "An implementation of fs-blob-store, which sorts files into alphabetical folders.",
"main": "index.js",
"dependencies": {},
"dependencies": {
"fs-blob-store": "^5.2.1",
"yargs": "^4.4.0"
},
"devDependencies": {
"abstract-blob-store": "^3.2.0",
"eslint": "^1.5.0",
"eslint-config-openstack": "^1.2.4",
"istanbul": "^0.4.2",
"jasmine": "^2.4.1"
"jasmine": "^2.4.1",
"nsp": "^2.3.0"
},
"scripts": {
"test": "istanbul cover jasmine",
"posttest": "istanbul check-coverage",
"prepublish": "nsp check",
"lint": "eslint ./"
},
"repository": {
"type": "git",
"url": "git://git.openstack.org/openstack/js-registry-static-afs"
"url": "git://git.openstack.org/openstack/js-sorting-fs-blob-store"
},
"keywords": [
"registry-static",
@ -28,5 +35,5 @@
"index.js",
"lib"
],
"author": "Michael Krotscheck <krotscheck@gmail.com> (http://www.krotscheck.net/)"
"author": "OpenStack <openstack-dev@lists.openstack.org> (http://www.openstack.org/)"
}