feat(sync): enable none syncModel proxy all public packages

Fixes #589 #581

config/index.js

@@ -164,17 +164,19 @@ var config = {
    * registry mode config
    */
 
-  // enable private mode, only admin can publish, other use just can sync package from source npm
-  enablePrivate: true,
+  // enable private mode or not
+  // private mode: only admins can publish, other users just can sync package from source npm
+  // public mode: all users can publish
+  enablePrivate: false,
 
   // registry scopes, if don't set, means do not support scopes
   scopes: [
     '@cnpm',
     '@cnpmtest'
   ],
 
-  // force user publish with scope
-  // but admins still can publish without scope
+  // force users publish with scope
+  // and admins still can publish without scope
   forcePublishWithScope: true,
 
   // some registry already have some private packages in global scope
@@ -205,7 +207,7 @@ var config = {
   syncByInstall: true,
 
   // sync mode select
-  // none: do not sync any module
+  // none: do not sync any module, proxy all public modules from sourceNpmRegistry
   // exist: only sync exist modules
   // all: sync all modules
   syncModel: 'exist', // 'none', 'all', 'exist'

controllers/registry/package/dist_tag.js

@@ -16,17 +16,6 @@
 
 var packageService = require('../../../services/package');
 
-function* getTags(ctx) {
-  var name = ctx.params.name || ctx.params[0];
-  var rows = yield* packageService.listModuleTags(name);
-  var tags = {};
-  for (var i = 0; i < rows.length; i++) {
-    var row = rows[i];
-    tags[row.tag] = row.version;
-  }
-  return tags;
-}
-
 function ok() {
   return {
     ok: "dist-tags updated"
@@ -35,7 +24,13 @@ function ok() {
 
 // GET /-/package/:pkg/dist-tags -- returns the package's dist-tags
 exports.index = function* () {
-  var tags = yield* getTags(this);
+  var name = this.params.name || this.params[0];
+  var rows = yield* packageService.listModuleTags(name);
+  var tags = {};
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    tags[row.tag] = row.version;
+  }
   this.body = tags;
 };
 

controllers/registry/package/list.js

@@ -16,10 +16,9 @@
 
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:list');
 var packageService = require('../../../services/package');
-var npmService = require('../../../services/npm');
-var config = require('../../../config');
-var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var common = require('../../../lib/common');
 var SyncModuleWorker = require('../../sync_module_worker');
+var utils = require('../../utils');
 
 /**
  * list all version of a module
@@ -114,23 +113,7 @@ module.exports = function* list() {
     var logId = yield* SyncModuleWorker.sync(name, 'sync-by-install');
     debug('start sync %s, get log id %s', name, logId);
 
-    // try to get package from official registry
-    var r = yield* npmService.request('/' + name, {
-      registry: config.officialNpmRegistry
-    });
-
-    debug('requet from officialNpmRegistry response %s', r.status);
-    if (r.status !== 200) {
-      this.status = 404;
-      this.body = {
-        error: 'not_found',
-        reason: 'document not found'
-      };
-      return;
-    }
-
-    this.body = r.data;
-    return;
+    return yield utils.proxyToNpmRegistry(this);
   }
 
   var latestMod = null;
@@ -150,7 +133,7 @@ module.exports = function* list() {
   for (var i = 0; i < rows.length; i++) {
     var row = rows[i];
     var pkg = row.package;
-    setDownloadURL(pkg, this);
+    common.setDownloadURL(pkg, this);
     pkg._cnpm_publish_time = row.publish_time;
     versions[pkg.version] = pkg;
 

controllers/sync_module_worker.js

@@ -146,6 +146,10 @@ SyncModuleWorker.prototype.start = function () {
       return;
     }
 
+    if (config.syncModel === 'none') {
+      return;
+    }
+
     var arr = [];
     for (var i = 0; i < that.concurrency; i++) {
       arr.push(that.next(i));
@@ -389,7 +393,7 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
   var mods = yield* packageService.listModulesByName(name);
   this.log('  [%s] start unpublished %d versions from local cnpm registry',
     name, mods.length);
-  if (this._isLocalModule(mods)) {
+  if (common.isLocalModule(mods)) {
     // publish on cnpm, dont sync this version package
     this.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -427,16 +431,6 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
   this.log('    [%s] delete nfs files: %j success', name, keys);
 };
 
-SyncModuleWorker.prototype._isLocalModule = function (mods) {
-  for (var i = 0; i < mods.length; i++) {
-    var r = mods[i];
-    if (r.package && r.package._publish_on_cnpm) {
-      return true;
-    }
-  }
-  return false;
-};
-
 SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var that = this;
   var hasModules = false;
@@ -451,7 +445,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var existsStarUsers = result[2];
   var existsNpmMaintainers = result[3];
 
-  if (that._isLocalModule(moduleRows)) {
+  if (common.isLocalModule(moduleRows)) {
     // publish on cnpm, dont sync this version package
     that.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];

controllers/utils.js

@@ -21,6 +21,7 @@ var utility = require('utility');
 var ms = require('humanize-ms');
 var moment = require('moment');
 var downloadTotalService = require('../services/download_total');
+var npmService = require('../services/npm');
 var nfs = require('../common/nfs');
 var config = require('../config');
 
@@ -159,3 +160,19 @@ exports.getOssLicenseUrlFromName = function (name) {
   return licenseMap[name.toLowerCase()] ?
     base + licenseMap[name.toLowerCase()] : base + name;
 };
+
+exports.proxyToNpmRegistry = function* (ctx) {
+  var r = yield* npmService.request(ctx.url);
+  if (!r.status && r.status < 200) {
+    r.status = 502;
+    r.body = {
+      error: 'remote_npm_registry_error',
+      reason: 'status: ' + r.status + ' body: ' + (r.data ? r.data.toString() : '[empty]'),
+    };
+    return;
+  }
+
+  ctx.status = r.status;
+  ctx.body = r.data;
+  return;
+};

lib/common.js

@@ -56,3 +56,13 @@ exports.isMaintainer = function (user, maintainers) {
 
   return match.length > 0;
 };
+
+exports.isLocalModule = function (mods) {
+  for (var i = 0; i < mods.length; i++) {
+    var r = mods[i];
+    if (r.package && r.package._publish_on_cnpm) {
+      return true;
+    }
+  }
+  return false;
+};

middleware/login.js

@@ -34,7 +34,7 @@ module.exports = function *login(next) {
     this.status = 401;
     this.body = {
       error: 'unauthorized',
-      reason: 'Login first.'
+      reason: 'Login first'
     };
     return;
   }

middleware/proxy_to_npm.js

@@ -0,0 +1,50 @@
+/**!
+ * cnpmjs.org - middleware/proxy_to_npm.js
+ *
+ * Copyright(c) Alibaba Group Holding Limited.
+ *
+ * Authors:
+ *   苏千 <[email protected]> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:middleware:proxy_to_npm');
+var config = require('../config');
+
+var proxyUrls = [
+  // /:pkg, dont contains scoped package
+  /^\/[\w\-\.]+$/,
+  // /-/package/:pkg/dist-tags
+  /^\/\-\/package\/[\w\-\.]+\/dist-tags/,
+];
+
+module.exports = function* proxyToNpm(next) {
+  if (config.syncModel !== 'none') {
+    return yield* next;
+  }
+  // only proxy read requests
+  if (this.method !== 'GET' && this.method !== 'HEAD') {
+    return yield* next;
+  }
+
+  var pathname = this.path;
+  var match;
+  for (var i = 0; i < proxyUrls.length; i++) {
+    match = proxyUrls[i].test(pathname);
+    if (match) {
+      break;
+    }
+  }
+  if (!match) {
+    return yield* next;
+  }
+
+  var url = config.sourceNpmRegistry + this.url;
+  debug('proxy to %s', url);
+  this.redirect(url);
+};

servers/registry.js

@@ -27,6 +27,7 @@ var auth = require('../middleware/auth');
 var staticCache = require('../middleware/static');
 var notFound = require('../middleware/registry_not_found');
 var cors = require('../middleware/cors');
+var proxyToNpm = require('../middleware/proxy_to_npm');
 
 app.use(block());
 middlewares.jsonp(app);
@@ -36,6 +37,7 @@ app.use(staticCache);
 
 app.keys = ['todokey', config.sessionSecret];
 app.proxy = true;
+app.use(proxyToNpm);
 app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(cors({
   methods: 'GET,HEAD'

services/npm.js

@@ -18,7 +18,7 @@
 var urllib = require('../common/urllib');
 var config = require('../config');
 
-var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+var USER_AGENT = 'npm_service.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
 function* request(url, options) {
   options = options || {};

test/controllers/registry/package/dist_tag.test.js

@@ -20,8 +20,9 @@ var pedding = require('pedding');
 var should = require('should');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
+var config = require('../../../../config');
 
-describe('controllers/registry/package.dist_tag.test.js', function () {
+describe('controllers/registry/package/dist_tag.test.js', function () {
   afterEach(mm.restore);
 
   describe('index()', function () {

test/controllers/registry/package/list.test.js

@@ -51,6 +51,15 @@ describe('controllers/registry/package/list.test.js', function () {
     });
   });
 
+  before(function (done) {
+    var worker = new SyncModuleWorker({
+      name: 'baidu',
+      noDep: true,
+    });
+    worker.start();
+    worker.on('end', done);
+  });
+
   it('should return all versions', function (done) {
     request(app.listen())
     .get('/testmodule-list-1')

test/middleware/proxy_to_npm.test.js

@@ -0,0 +1,65 @@
+/**!
+ * cnpmjs.org - test/middleware/proxy_to_npm.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <[email protected]> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var request = require('supertest');
+var mm = require('mm');
+var app = require('../../servers/registry');
+var config = require('../../config');
+
+describe('middleware/proxy_to_npm.test.js', function () {
+  beforeEach(function () {
+    mm(config, 'syncModel', 'none');
+  });
+
+  afterEach(mm.restore);
+
+  describe('package', function () {
+    it('should proxy to source registry when package not exists', function (done) {
+      request(app.listen())
+      .get('/ms')
+      .expect('location', config.sourceNpmRegistry + '/ms')
+      .expect(302, done);
+    });
+
+    it('should proxy to source registry when package is not local', function (done) {
+      request(app.listen())
+      .get('/baidu?write=true')
+      .expect('location', config.sourceNpmRegistry + '/baidu?write=true')
+      .expect(302, done);
+    });
+
+    it('should not proxy to source registry when package is scoped', function (done) {
+      request(app.listen())
+      .get('/@scoped/test-package-name')
+      .expect(404, done);
+    });
+  });
+
+  describe('dist-tags', function () {
+    it('should proxy to source registry when package not exists', function (done) {
+      request(app.listen())
+      .get('/-/package/ms/dist-tags')
+      .expect('location', config.sourceNpmRegistry + '/-/package/ms/dist-tags')
+      .expect(302, done);
+    });
+
+    it('should dont proxy scoped package', function (done) {
+      request(app.listen())
+      .get('/-/package/@scoped/ms/dist-tags')
+      .expect(404, done);
+    });
+  });
+});