Merge pull request #16462 from mozilla/fxa-redis-v2

feat(redis): Add support to customs server to use redis or memcache

packages/fxa-customs-server/lib/cache.js

@@ -0,0 +1,50 @@
+const Memcached = require('memcached');
+const { RedisShared } = require('fxa-shared/db/redis');
+const P = require('bluebird');
+P.promisifyAll(Memcached.prototype);
+
+class Cache {
+  constructor(config) {
+    const customsRedisConfig = config.redis.customs;
+    this.useRedis = customsRedisConfig.enabled;
+
+    if (this.useRedis) {
+      this.client = new RedisShared(config.redis.customs);
+    } else {
+      this.client = new Memcached(config.memcache.address, {
+        timeout: 500,
+        retries: 1,
+        retry: 1000,
+        reconnect: 1000,
+        idle: 30000,
+        namespace: 'fxa~',
+      });
+    }
+  }
+
+  async setAsync(key, value, lifetime) {
+    if (this.useRedis) {
+      if (lifetime === 0) {
+        return this.client.redis.set(key, JSON.stringify(value));
+      }
+      // Set the value in redis. We use 'EX' to set the expiration time in seconds.
+      return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime);
+    }
+    return this.client.setAsync(key, value, lifetime);
+  }
+
+  async getAsync(key) {
+    if (this.useRedis) {
+      const value = await this.client.redis.get(key);
+      try {
+        return JSON.parse(value);
+      } catch (err) {
+        return {};
+      }
+    } else {
+      return this.client.getAsync(key);
+    }
+  }
+}
+
+module.exports = Cache;

packages/fxa-customs-server/lib/config/config.js

@@ -3,6 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 const { tracingConfig } = require('fxa-shared/tracing/config');
+const { makeRedisConfig } = require('fxa-shared/db/config');
 
 module.exports = function (fs, path, url, convict) {
   var conf = convict({
@@ -186,6 +187,7 @@ module.exports = function (fs, path, url, convict) {
         env: 'RECORD_LIFETIME_SECONDS',
       },
     },
+    redis: makeRedisConfig(),
     allowedIPs: {
       doc: 'An array of IPs that will not be blocked or rate-limited.',
       format: Array,

packages/fxa-customs-server/lib/config/dev.json

@@ -0,0 +1,7 @@
+{
+  "redis": {
+    "customs": {
+      "enabled": false
+    }
+  }
+}

packages/fxa-customs-server/lib/server.js

@@ -8,16 +8,16 @@
 
 const Hapi = require('@hapi/hapi');
 const HapiSwagger = require('hapi-swagger');
-const Memcached = require('memcached');
 const swaggerOptions = require('../docs/swagger/swagger-options');
 const API_DOCS = require('../docs/swagger/customs-server-api');
 const packageJson = require('../package.json');
 const blockReasons = require('./block_reasons');
 const P = require('bluebird');
-P.promisifyAll(Memcached.prototype);
+
 const { configureSentry } = require('./sentry');
 const dataflow = require('./dataflow');
 const { StatsD } = require('hot-shots');
+const Cache = require('./cache');
 
 module.exports = async function createServer(config, log) {
   var startupDefers = [];
@@ -47,14 +47,7 @@ module.exports = async function createServer(config, log) {
         timing: () => {},
       };
 
-  var mc = new Memcached(config.memcache.address, {
-    timeout: 500,
-    retries: 1,
-    retry: 1000,
-    reconnect: 1000,
-    idle: 30000,
-    namespace: 'fxa~',
-  });
+  const mc = new Cache(config);
 
   var reputationService = require('./reputationService')(config, log);
   const Settings = require('./settings/settings')(config, mc, log);

packages/fxa-customs-server/package.json

@@ -23,6 +23,7 @@
     "test": "scripts/test-local.sh",
     "test-unit": "yarn make-artifacts-dir && tap test/local --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-unit.xml",
     "test-integration": "yarn make-artifacts-dir && tap test/remote test/scripts --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration.xml",
+    "test-integration-redis": "CUSTOMS_REDIS_ENABLED=true yarn make-artifacts-dir && tap test/remote test/scripts --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration-redis.xml",
     "format": "prettier --write --config ../../_dev/.prettierrc '**'",
     "make-artifacts-dir": "mkdir -p ../../artifacts/tests/$npm_package_name"
   },

packages/fxa-customs-server/scripts/test-local.sh

@@ -4,4 +4,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+echo 'Testing with Memcache database'
 tap test/local test/remote test/scripts --no-coverage --jobs=1
+
+echo 'Testing with Redis database'
+CUSTOMS_REDIS_ENABLED=true tap test/local test/remote test/scripts --no-coverage --jobs=1

packages/fxa-customs-server/test/memcache-helper.js

@@ -5,13 +5,21 @@
 'use strict';
 
 var P = require('bluebird');
-var Memcached = require('memcached');
-P.promisifyAll(Memcached.prototype);
+const Cache = require('../lib/cache');
 
 var config = {
   memcache: {
     address: process.env.MEMCACHE_ADDRESS || 'localhost:11211',
   },
+  redis: {
+    customs: {
+      enabled: process.env.CUSTOMS_REDIS_ENABLED === 'true',
+      host: 'localhost',
+      password: '',
+      port: 6379,
+      prefix: 'customs:',
+    },
+  },
   limits: {
     blockIntervalSeconds: 1,
     rateLimitIntervalSeconds: 1,
@@ -43,14 +51,7 @@ var config = {
   },
 };
 
-var mc = new Memcached(config.memcache.address, {
-  timeout: 500,
-  retries: 1,
-  retry: 1000,
-  reconnect: 1000,
-  idle: 30000,
-  namespace: 'fxa~',
-});
+var mc = new Cache(config);
 
 module.exports.mc = mc;
 
@@ -85,22 +86,35 @@ var IpRecord = require('../lib/ip_record')(limits);
 
 module.exports.limits = limits;
 
-function blockedEmailCheck(cb) {
-  setTimeout(
-    // give memcache time to flush the writes
-    function () {
-      mc.get(TEST_EMAIL, function (err, data) {
-        var er = EmailRecord.parse(data);
-        mc.end();
-        cb(er.shouldBlock());
-      });
-    }
-  );
+async function blockedEmailCheck(email, cb) {
+  if (config.redis.customs.enabled) {
+    return P.resolve(true).then(async () => {
+      const result = await mc.getAsync(email);
+      var er = EmailRecord.parse(result);
+      cb(er.shouldBlock());
+    });
+  }
+  // give memcache time to flush the writes
+  setTimeout(function () {
+    mc.client.get(email, function (err, data) {
+      var er = EmailRecord.parse(data);
+      mc.client.end();
+      cb(er.shouldBlock());
+    });
+  });
 }
 
 module.exports.blockedEmailCheck = blockedEmailCheck;
 
 function blockedIpCheck(cb) {
+  if (config.redis.customs.enabled) {
+    return Promise.resolve(true).then(async () => {
+      const result = await mc.getAsync(TEST_IP);
+      var er = IpRecord.parse(result);
+      cb(er.shouldBlock());
+    });
+  }
+
   setTimeout(
     // give memcache time to flush the writes
     function () {
@@ -124,7 +138,7 @@ function badLoginCheck() {
     var ipEmailRecord = IpEmailRecord.parse(d1);
     var ipRecord = IpRecord.parse(d2);
     var emailRecord = EmailRecord.parse(d3);
-    mc.end();
+    mc.client.end();
     return {
       ipEmailRecord: ipEmailRecord,
       ipRecord: ipRecord,
@@ -136,7 +150,17 @@ function badLoginCheck() {
 module.exports.badLoginCheck = badLoginCheck;
 
 function clearEverything(cb) {
-  mc.itemsAsync()
+  if (config.redis.customs.enabled) {
+    return mc.client.redis.flushall(function (err) {
+      if (err) {
+        return cb(err);
+      }
+      cb();
+    });
+  }
+
+  mc.client
+    .itemsAsync()
     .then(function (result) {
       var firstServer = result[0];
 
@@ -148,7 +172,7 @@ function clearEverything(cb) {
       // get a cachedump for each slabid and slab.number
       var cachedumps = keys
         .map(function (stats) {
-          return mc.cachedumpAsync(
+          return mc.client.cachedumpAsync(
             firstServer.server,
             stats,
             firstServer[stats].number
@@ -167,7 +191,7 @@ function clearEverything(cb) {
               dump
                 .filter((item) => /^fxa~/.test(item.key))
                 .map(function (item) {
-                  return mc.delAsync(item.key.replace(/^fxa~/, ''));
+                  return mc.client.delAsync(item.key.replace(/^fxa~/, ''));
                 })
             );
           });
@@ -176,7 +200,7 @@ function clearEverything(cb) {
       return P.all(cachedumps);
     })
     .then(function () {
-      mc.end();
+      mc.client.end();
       cb();
     })
     .catch(cb);
@@ -191,7 +215,7 @@ function setLimits(settings) {
     limits[k] = settings[k];
   }
   return limits.push().then(function (s) {
-    mc.end();
+    mc.client.end();
     return s;
   });
 }
@@ -201,7 +225,7 @@ module.exports.setLimits = setLimits;
 function setAllowedIPs(ips) {
   allowedIPs.setAll(ips);
   return allowedIPs.push().then(function (ips) {
-    mc.end();
+    mc.client.end();
     return ips;
   });
 }
@@ -211,7 +235,7 @@ module.exports.setAllowedIPs = setAllowedIPs;
 function setAllowedEmailDomains(domains) {
   allowedEmailDomains.setAll(domains);
   return allowedEmailDomains.push().then(function (domains) {
-    mc.end();
+    mc.client.end();
     return domains;
   });
 }
@@ -221,7 +245,7 @@ module.exports.setAllowedEmailDomains = setAllowedEmailDomains;
 function setAllowedPhoneNumbers(phoneNumbers) {
   allowedPhoneNumbers.setAll(phoneNumbers);
   return allowedPhoneNumbers.push().then((phoneNumbers) => {
-    mc.end();
+    mc.client.end();
     return phoneNumbers;
   });
 }
@@ -237,9 +261,13 @@ function setRequestChecks(settings) {
     requestChecks[k] = settings[k];
   }
   return requestChecks.push().then(function (s) {
-    mc.end();
+    mc.client.end();
     return s;
   });
 }
 
 module.exports.setAllowedEmailDomains = setAllowedEmailDomains;
+
+if (config.redis.customs.enabled) {
+  mc.client.end = function () {};
+}

packages/fxa-customs-server/test/remote/block_email_tests.js

@@ -5,13 +5,14 @@ var test = require('tap').test;
 var restifyClients = require('restify-clients');
 var TestServer = require('../test_server');
 var Promise = require('bluebird');
-var mcHelper = require('../memcache-helper');
+const { randomIp, randomEmail } = require('../utils');
 
-var TEST_EMAIL = '[email protected]';
+var TEST_EMAIL = randomEmail();
 var ALLOWED_EMAIL = '[email protected]';
-var TEST_IP = '192.0.2.1';
+var TEST_IP = randomIp();
 
 const config = require('../../lib/config').getProperties();
+
 var testServer = new TestServer(config);
 
 var client = restifyClients.createJsonClient({
@@ -26,13 +27,6 @@ test('startup', async function (t) {
   t.end();
 });
 
-test('clear everything', function (t) {
-  mcHelper.clearEverything(function (err) {
-    t.notOk(err, 'no errors were returned');
-    t.end();
-  });
-});
-
 test('missing email', function (t) {
   return client.postAsync('/blockEmail', {}).then(
     function (req, res, obj) {

packages/fxa-customs-server/test/remote/check_ip_only_tests.js

@@ -3,20 +3,15 @@
 
 'use strict';
 
-const memcached = require('../memcache-helper');
 const restifyClients = require('restify-clients');
-const test = require('tap').test;
+const { randomIp } = require('../utils');
 const TestServer = require('../test_server');
+const test = require('tap').test;
 
-const IP = '192.168.1.1';
+const IP = randomIp();
 const ACTION = 'wibble';
 
-const config = {
-  listen: {
-    port: 7000,
-  },
-};
-
+const config = require('../../lib/config').getProperties();
 const testServer = new TestServer(config);
 
 const client = restifyClients.createJsonClient({
@@ -29,13 +24,6 @@ test('startup', async function (t) {
   t.end();
 });
 
-test('clear everything', (t) => {
-  memcached.clearEverything((err) => {
-    t.notOk(err, 'memcached.clearEverything should not return an error');
-    t.end();
-  });
-});
-
 test('with ip and action', (t) => {
   client.post(
     '/checkIpOnly',