diff --git a/.circleci/config.yml b/.circleci/config.yml index 8537fb2298..fce529e19f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,7 +119,6 @@ executors: - image: cimg/mysql:8.0 command: --default-authentication-plugin=mysql_native_password - image: jdlk7/firestore-emulator - - image: memcached - image: redis - image: ghcr.io/aertje/cloud-tasks-emulator:1.2.0 command: -queue "projects/test/locations/test/queues/delete-accounts-queue" @@ -141,7 +140,6 @@ executors: docker: - image: mozilla/fxa-circleci:ci-functional-test-runner-v3 - image: redis - - image: memcached - image: pafortin/goaws - image: cimg/mysql:8.0 command: --default-authentication-plugin=mysql_native_password diff --git a/_dev/docker/docker-compose.yml b/_dev/docker/docker-compose.yml index b4a354c185..1764dd93ce 100644 --- a/_dev/docker/docker-compose.yml +++ b/_dev/docker/docker-compose.yml @@ -31,7 +31,6 @@ services: - DB=mysql - CONFIG_FILES=config/secrets.json - MYSQL_HOST=mysql - - MEMCACHE_METRICS_CONTEXT_ADDRESS=memcached:11211 - HTTPDB_URL=http://authdb:8000 - EMAIL_SERVICE_HOST=email - REDIS_HOST=redis @@ -80,7 +79,6 @@ services: - "1111:1111" depends_on: - mysql - - memcached profile-worker: image: fxa-profile-server:build entrypoint: /bin/bash -c @@ -99,7 +97,6 @@ services: - "1113:1113" depends_on: - mysql - - memcached profile-static: image: fxa-profile-server:build entrypoint: /bin/bash -c @@ -116,7 +113,6 @@ services: - "1112:1112" depends_on: - mysql - - memcached redis: image: redis mysql: @@ -125,8 +121,6 @@ services: - MYSQL_ALLOW_EMPTY_PASSWORD=true - MYSQL_ROOT_HOST=% - MYSQL_DATABASE=fxa - memcached: - image: memcached maildev: image: djfarrelly/maildev ports: diff --git a/_dev/pm2/infrastructure.config.js b/_dev/pm2/infrastructure.config.js index 21ef6998f7..95590a5c6e 100644 --- a/_dev/pm2/infrastructure.config.js +++ b/_dev/pm2/infrastructure.config.js @@ -17,13 +17,6 @@ module.exports = { min_uptime: '2m', kill_timeout: 20000, }, - { - name: 'memcache', - script: '_scripts/memcached.sh', - max_restarts: '1', - min_uptime: '2m', - kill_timeout: 20000, - }, { name: 'sns', script: '_scripts/goaws.sh', diff --git a/_scripts/check-ports.sh b/_scripts/check-ports.sh index d74855f365..2e828a0dc9 100755 --- a/_scripts/check-ports.sh +++ b/_scripts/check-ports.sh @@ -3,7 +3,6 @@ PORTS=( 3306 # MySQL server 6379 # redis - 11211 # memcached 4100 # Fake SQS/SNS 8085 # google-pubsub-emulator 9090 # google-firestore-emulator diff --git a/_scripts/memcached.sh b/_scripts/memcached.sh deleted file mode 100755 index af26d9fb8b..0000000000 --- a/_scripts/memcached.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -ex - -docker run --rm --net fxa -p 11211:11211 --name memcache memcached diff --git a/backstage/memcached.resources.yaml b/backstage/memcached.resources.yaml deleted file mode 100644 index 3de9d4016c..0000000000 --- a/backstage/memcached.resources.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: fxa-customs-cache - description: Caches rate limits and IP reuptation scores. - tags: - - memcached -spec: - type: cache - owner: fxa-devs - system: mozilla-accounts diff --git a/backstage/redis.resources.yaml b/backstage/redis.resources.yaml index a3d3a295a4..5605076ee1 100644 --- a/backstage/redis.resources.yaml +++ b/backstage/redis.resources.yaml @@ -22,3 +22,15 @@ spec: type: cache owner: fxa-devs system: mozilla-accounts +--- +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: fxa-customs-cache + description: Caches rate limits and IP reputation scores. + tags: + - redis +spec: + type: cache + owner: fxa-devs + system: mozilla-accounts diff --git a/catalog-info.yaml b/catalog-info.yaml index ec081306ff..19a2e570d1 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -10,7 +10,6 @@ spec: - ./backstage/mozilla-accounts.system.yaml - ./backstage/mysql.resources.yaml - ./backstage/redis.resources.yaml - - ./backstage/memcached.resources.yaml - ./packages/123done/backstage.yaml - ./packages/browserid-verifier/backstage.yaml - ./packages/fxa-admin-panel/backstage.yaml diff --git a/packages/fxa-auth-server/README.md b/packages/fxa-auth-server/README.md index 605fbc1f8e..1bdf723b74 100644 --- a/packages/fxa-auth-server/README.md +++ b/packages/fxa-auth-server/README.md @@ -70,7 +70,7 @@ We also use [Chai](https://www.chaijs.com/) for making assertions in tests. As o ### Testing with non-local databases -Executing tests using remote databases (MySQL, Redis, Memcached) is possible by specifying (and exporting) the following environment variables: +Executing tests using remote databases (MySQL, Redis) is possible by specifying (and exporting) the following environment variables: - MySQL: - MYSQL_HOST @@ -80,8 +80,7 @@ Executing tests using remote databases (MySQL, Redis, Memcached) is possible by - REDIS_HOST - ACCESS_TOKEN_REDIS_HOST - REFRESH_TOKEN_REDIS_HOST -- Memcached: - - MEMCACHE_METRICS_CONTEXT_ADDRESS + - METRICS_REDIS_HOST This also allows to use temporary throw-away Docker containers to provide these. diff --git a/packages/fxa-auth-server/package.json b/packages/fxa-auth-server/package.json index 9172147e6d..254d447ed1 100644 --- a/packages/fxa-auth-server/package.json +++ b/packages/fxa-auth-server/package.json @@ -160,7 +160,6 @@ "@types/jws": "^3.2.3", "@types/lodash": "^4", "@types/luxon": "^3", - "@types/memcached": "^2.2.6", "@types/mocha": "^10.0.6", "@types/nock": "^11.1.0", "@types/node": "^20.11.1", diff --git a/packages/fxa-auth-server/test/local/metrics/context.js b/packages/fxa-auth-server/test/local/metrics/context.js index 6380898d48..83d8a22ace 100644 --- a/packages/fxa-auth-server/test/local/metrics/context.js +++ b/packages/fxa-auth-server/test/local/metrics/context.js @@ -658,7 +658,7 @@ describe('metricsContext', () => { .catch((err) => assert.fail(err)); }); - it('metricsContext.clear with memcached error', () => { + it('metricsContext.clear with cache error', () => { const token = { uid: Array(64).fill('7').join(''), id: 'wibble', @@ -677,7 +677,7 @@ describe('metricsContext', () => { assert.equal( err.message, 'blee', - 'metricsContext.clear should have rejected with memcached error' + 'metricsContext.clear should have rejected with cache error' ); assert.equal(cache.del.callCount, 1, 'cache.del was called once'); }); diff --git a/packages/fxa-auth-server/test/local/routes/account.js b/packages/fxa-auth-server/test/local/routes/account.js index 25c46503bd..251e91a51a 100644 --- a/packages/fxa-auth-server/test/local/routes/account.js +++ b/packages/fxa-auth-server/test/local/routes/account.js @@ -54,11 +54,6 @@ const makeRoutes = function (options = {}, requireMocks = {}) { config.oauth = config.oauth || {}; config.verifierVersion = config.verifierVersion || 0; config.smtp = config.smtp || {}; - config.memcached = config.memcached || { - address: 'none', - idle: 500, - lifetime: 30, - }; config.lastAccessTimeUpdates = {}; config.signinConfirmation = config.signinConfirmation || {}; config.signinUnblock = config.signinUnblock || {}; diff --git a/packages/fxa-auth-server/test/local/routes/attached-clients.js b/packages/fxa-auth-server/test/local/routes/attached-clients.js index 3d6f8ca9c7..4e57232080 100644 --- a/packages/fxa-auth-server/test/local/routes/attached-clients.js +++ b/packages/fxa-auth-server/test/local/routes/attached-clients.js @@ -23,11 +23,6 @@ const mockAuthorizedClients = { function makeRoutes(options = {}) { const config = options.config || {}; config.smtp = config.smtp || {}; - config.memcached = config.memcached || { - address: 'localhost:1121', - idle: 500, - lifetime: 30, - }; config.i18n = { supportedLanguages: ['en', 'fr'], defaultLanguage: 'en', @@ -378,9 +373,9 @@ describe('/account/attached_clients', () => { it('filters out idle devices', async () => { const now = Date.now(); - const FIVE_DAYS_AGO = now - (1000 * 60 * 60 * 24 * 5); - const ONE_DAY_AGO = now - (1000 * 60 * 60 * 24); - + const FIVE_DAYS_AGO = now - 1000 * 60 * 60 * 24 * 5; + const ONE_DAY_AGO = now - 1000 * 60 * 60 * 24; + request.query.filterIdleDevicesTimestamp = ONE_DAY_AGO; // Filter for devices active in the last day const DEVICES = [ { @@ -402,7 +397,7 @@ describe('/account/attached_clients', () => { createdAt: now, lastAccessTime: now, location: { country: 'Germany' }, - } + }, ]; const OAUTH_CLIENTS = []; diff --git a/packages/fxa-auth-server/test/local/routes/devices-and-sessions.js b/packages/fxa-auth-server/test/local/routes/devices-and-sessions.js index 1589f5600b..0210ed3c7a 100644 --- a/packages/fxa-auth-server/test/local/routes/devices-and-sessions.js +++ b/packages/fxa-auth-server/test/local/routes/devices-and-sessions.js @@ -21,11 +21,6 @@ function makeRoutes(options = {}, requireMocks) { const config = options.config || {}; config.oauth = config.oauth || {}; config.smtp = config.smtp || {}; - config.memcached = config.memcached || { - address: 'localhost:1121', - idle: 500, - lifetime: 30, - }; config.i18n = { supportedLanguages: ['en', 'fr'], defaultLanguage: 'en', diff --git a/packages/fxa-auth-server/test/local/routes/emails.js b/packages/fxa-auth-server/test/local/routes/emails.js index 5ee82f6aa2..71388dc62e 100644 --- a/packages/fxa-auth-server/test/local/routes/emails.js +++ b/packages/fxa-auth-server/test/local/routes/emails.js @@ -153,11 +153,6 @@ const makeRoutes = function (options = {}, requireMocks) { const config = options.config || {}; config.verifierVersion = config.verifierVersion || 0; config.smtp = config.smtp || {}; - config.memcached = config.memcached || { - address: 'none', - idle: 500, - lifetime: 30, - }; config.i18n = { supportedLanguages: ['en'], defaultLanguage: 'en', diff --git a/packages/fxa-customs-server/.awsbox.json b/packages/fxa-customs-server/.awsbox.json deleted file mode 100644 index 08ccdbf86d..0000000000 --- a/packages/fxa-customs-server/.awsbox.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "processes": [ - "bin/customs_server.js" - ], - "packages": [ - "memcached" - ], - "ssl": "disable" -} diff --git a/packages/fxa-customs-server/README.md b/packages/fxa-customs-server/README.md index 80c5b60d65..4b808dbce4 100644 --- a/packages/fxa-customs-server/README.md +++ b/packages/fxa-customs-server/README.md @@ -63,7 +63,7 @@ We currently have the following policies in place: - manual blocking of an account (see `/blockEmail` API call) - manual blocking of an IP address (see `/blockIp` API call) -The data that these policies are based on is stored in a memcache instance (keyed by `email`, `ip` or `ip + email` depending on the policy) and the code that implements them is split across these three files: +The data that these policies are based on is stored in a redis cache instance (keyed by `email`, `ip` or `ip + email` depending on the policy) and the code that implements them is split across these three files: - `email_record.js` handles blocking and rate-limiting based only on the email address - `ip_email_record.js` handles rate-limiting based on both the email and IP address of the request diff --git a/packages/fxa-customs-server/lib/cache.js b/packages/fxa-customs-server/lib/cache.js index 1754bc8f42..75fc735e2e 100644 --- a/packages/fxa-customs-server/lib/cache.js +++ b/packages/fxa-customs-server/lib/cache.js @@ -1,48 +1,28 @@ -const Memcached = require('memcached'); +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + const { RedisShared } = require('fxa-shared/db/redis'); -const P = require('bluebird'); -P.promisifyAll(Memcached.prototype); class Cache { constructor(config) { - const customsRedisConfig = config.redis.customs; - this.useRedis = customsRedisConfig.enabled; - - if (this.useRedis) { - this.client = new RedisShared(config.redis.customs); - } else { - this.client = new Memcached(config.memcache.address, { - timeout: 500, - retries: 1, - retry: 1000, - reconnect: 1000, - idle: 30000, - namespace: 'fxa~', - }); - } + this.client = new RedisShared(config.redis.customs); } async setAsync(key, value, lifetime) { - if (this.useRedis) { - if (lifetime === 0) { - return this.client.redis.set(key, JSON.stringify(value)); - } - // Set the value in redis. We use 'EX' to set the expiration time in seconds. - return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime); + if (lifetime === 0) { + return this.client.redis.set(key, JSON.stringify(value)); } - return this.client.setAsync(key, value, lifetime); + // Set the value in redis. We use 'EX' to set the expiration time in seconds. + return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime); } async getAsync(key) { - if (this.useRedis) { - const value = await this.client.redis.get(key); - try { - return JSON.parse(value); - } catch (err) { - return {}; - } - } else { - return this.client.getAsync(key); + const value = await this.client.redis.get(key); + try { + return JSON.parse(value); + } catch (err) { + return {}; } } } diff --git a/packages/fxa-customs-server/lib/config/config.js b/packages/fxa-customs-server/lib/config/config.js index 9218f67c39..08ce371164 100644 --- a/packages/fxa-customs-server/lib/config/config.js +++ b/packages/fxa-customs-server/lib/config/config.js @@ -174,14 +174,9 @@ module.exports = function (fs, path, url, convict) { env: 'MAX_ACCOUNT_ACCESS', }, }, - memcache: { - address: { - doc: 'Hostname/IP:Port of the memcache server', - default: 'localhost:11211', - env: 'MEMCACHE_ADDRESS', - }, + cache: { recordLifetimeSeconds: { - doc: 'Memcache record expiry', + doc: 'Record expiry', default: 900, format: 'nat', env: 'RECORD_LIFETIME_SECONDS', diff --git a/packages/fxa-customs-server/lib/config/dev.json b/packages/fxa-customs-server/lib/config/dev.json index 3d4b38c168..72f7e16f83 100644 --- a/packages/fxa-customs-server/lib/config/dev.json +++ b/packages/fxa-customs-server/lib/config/dev.json @@ -1,7 +1,7 @@ { "redis": { "customs": { - "enabled": false + "enabled": true } } } \ No newline at end of file diff --git a/packages/fxa-customs-server/lib/record.js b/packages/fxa-customs-server/lib/record.js index a15030c79d..70a4074f22 100644 --- a/packages/fxa-customs-server/lib/record.js +++ b/packages/fxa-customs-server/lib/record.js @@ -9,14 +9,14 @@ class Record { this.hits = object.hits || []; // timestamps when last hit occurred Object.defineProperty(this, 'limits', { - // limits is not saved to memcached + // limits is not saved to cache enumerable: false, get() { return config.limits; }, }); Object.defineProperty(this, 'actions', { - // actions is not saved to memcached + // actions is not saved to cache enumerable: false, get() { return config.actions; diff --git a/packages/fxa-customs-server/lib/server.js b/packages/fxa-customs-server/lib/server.js index 28526b3c9d..21850d5199 100755 --- a/packages/fxa-customs-server/lib/server.js +++ b/packages/fxa-customs-server/lib/server.js @@ -74,7 +74,7 @@ module.exports = async function createServer(config, log) { mc, reputationService, limits, - config.memcache.recordLifetimeSeconds, + config.cache.recordLifetimeSeconds, statsd ); @@ -123,7 +123,7 @@ module.exports = async function createServer(config, log) { await configureSentry(api, config, log); function logError(err) { - log.error({ op: 'memcachedError', err: err }); + log.error({ op: 'cacheError', err: err }); throw err; } function normalizePath(path) { diff --git a/packages/fxa-customs-server/lib/settings/allowed_email_domains.js b/packages/fxa-customs-server/lib/settings/allowed_email_domains.js index 2761074bfb..8fd47ea068 100644 --- a/packages/fxa-customs-server/lib/settings/allowed_email_domains.js +++ b/packages/fxa-customs-server/lib/settings/allowed_email_domains.js @@ -30,7 +30,7 @@ module.exports = (config, Settings, log) => { op: 'allowedEmailDomains.validate.invalid', data: domains, }); - throw new Settings.Missing('invalid allowedEmailDomains from memcache'); + throw new Settings.Missing('invalid allowedEmailDomains from cache'); } return domains; } diff --git a/packages/fxa-customs-server/lib/settings/allowed_ips.js b/packages/fxa-customs-server/lib/settings/allowed_ips.js index b173ea72c7..20bc3efb6a 100644 --- a/packages/fxa-customs-server/lib/settings/allowed_ips.js +++ b/packages/fxa-customs-server/lib/settings/allowed_ips.js @@ -28,7 +28,7 @@ module.exports = (config, Settings, log) => { validate(ips) { if (!Array.isArray(ips)) { log.error({ op: 'allowedIPs.validate.invalid', data: ips }); - throw new Settings.Missing('invalid allowedIPs from memcache'); + throw new Settings.Missing('invalid allowedIPs from cache'); } return ips.filter(function (ip) { var is = net.isIPv4(ip); diff --git a/packages/fxa-customs-server/lib/settings/allowed_phone_numbers.js b/packages/fxa-customs-server/lib/settings/allowed_phone_numbers.js index e496c2a48e..e3546c9f17 100644 --- a/packages/fxa-customs-server/lib/settings/allowed_phone_numbers.js +++ b/packages/fxa-customs-server/lib/settings/allowed_phone_numbers.js @@ -26,7 +26,7 @@ module.exports = (config, Settings, log) => { op: 'allowedPhoneNumbers.validate.invalid', data: phoneNumbers, }); - throw new Settings.Missing('invalid allowedPhoneNumbers from memcache'); + throw new Settings.Missing('invalid allowedPhoneNumbers from cache'); } return phoneNumbers; } diff --git a/packages/fxa-customs-server/lib/settings/limits.js b/packages/fxa-customs-server/lib/settings/limits.js index 47bd612038..43be0f180b 100644 --- a/packages/fxa-customs-server/lib/settings/limits.js +++ b/packages/fxa-customs-server/lib/settings/limits.js @@ -53,7 +53,7 @@ module.exports = (config, Settings, log) => { validate(settings) { if (typeof settings !== 'object') { log.error({ op: 'limits.validate.invalid', data: settings }); - throw new Settings.Missing('invalid limits from memcache'); + throw new Settings.Missing('invalid limits from cache'); } var keys = Object.keys(config.limits); for (var i = 0; i < keys.length; i++) { diff --git a/packages/fxa-customs-server/lib/settings/requestChecks.js b/packages/fxa-customs-server/lib/settings/requestChecks.js index 6358b1a533..e908e636ed 100644 --- a/packages/fxa-customs-server/lib/settings/requestChecks.js +++ b/packages/fxa-customs-server/lib/settings/requestChecks.js @@ -21,8 +21,8 @@ module.exports = (config, Settings, log) => { if (!this.flowIdExemptUserAgentREs) { this.flowIdExemptUserAgentCompiledREs = []; } else { - this.flowIdExemptUserAgentCompiledREs = this.flowIdExemptUserAgentREs.map( - function (re) { + this.flowIdExemptUserAgentCompiledREs = + this.flowIdExemptUserAgentREs.map(function (re) { // Log the regex at startup to check that we have correct values. const regex = new RegExp(re); log.info({ @@ -30,8 +30,7 @@ module.exports = (config, Settings, log) => { regex: String(regex), }); return regex; - } - ); + }); } return this; } @@ -41,7 +40,7 @@ module.exports = (config, Settings, log) => { validate(settings) { if (typeof settings !== 'object') { log.error({ op: 'requestChecks.validate.invalid', data: settings }); - throw new Settings.Missing('invalid requestChecks from memcache'); + throw new Settings.Missing('invalid requestChecks from cache'); } const keys = Object.keys(config.requestChecks); for (let i = 0; i < keys.length; i++) { diff --git a/packages/fxa-customs-server/lib/settings/settings.js b/packages/fxa-customs-server/lib/settings/settings.js index 2faa67a805..3b7e7745f9 100644 --- a/packages/fxa-customs-server/lib/settings/settings.js +++ b/packages/fxa-customs-server/lib/settings/settings.js @@ -11,12 +11,12 @@ const POLL_INTERVAL = Symbol(); module.exports = (config, mc, log) => { // A sentinel error for signaling that the result was invalid/missing, - // and we should try to push our own representation to memcached. + // and we should try to push our own representation to cache. class Missing extends Error {} - // An abstract class that stores options in memcached. + // An abstract class that stores options in cache. // - // Others extend this class to hotload options from memcached. + // Others extend this class to hotload options from cache. class Settings { constructor(key) { assert(typeof key === 'string'); @@ -61,7 +61,7 @@ module.exports = (config, mc, log) => { return result.then( (value) => this.setAll(value), (err) => { - log.error({ op: this[KEY] + '.refresh', err: err.message}); + log.error({ op: this[KEY] + '.refresh', err: err.message }); throw err; } ); diff --git a/packages/fxa-customs-server/package.json b/packages/fxa-customs-server/package.json index 8d4eef5a6f..b8d7058321 100644 --- a/packages/fxa-customs-server/package.json +++ b/packages/fxa-customs-server/package.json @@ -22,8 +22,7 @@ "lint": "eslint .", "test": "scripts/test-local.sh", "test-unit": "yarn make-artifacts-dir && tap test/local --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-unit.xml", - "test-integration": "yarn make-artifacts-dir && tap test/remote test/scripts --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration.xml", - "test-integration-redis": "CUSTOMS_REDIS_ENABLED=true yarn make-artifacts-dir && tap test/remote test/scripts --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration-redis.xml", + "test-integration": "yarn make-artifacts-dir && tap test/remote --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration.xml", "format": "prettier --write --config ../../_dev/.prettierrc '**'", "make-artifacts-dir": "mkdir -p ../../artifacts/tests/$npm_package_name" }, @@ -44,8 +43,7 @@ "ip": "^2.0.1", "ip-reputation-js-client": "^6.0.4", "lodash.isequal": "4.5.0", - "lodash.merge": "4.6.2", - "memcached": "^2.2.2" + "lodash.merge": "4.6.2" }, "devDependencies": { "@types/dedent": "^0", diff --git a/packages/fxa-customs-server/scripts/block-ip.js b/packages/fxa-customs-server/scripts/block-ip.js deleted file mode 100755 index 63e7675643..0000000000 --- a/packages/fxa-customs-server/scripts/block-ip.js +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env node -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/*/ - -Block an ip address for a number of seconds. - -Example: - -$ ./block-ip.js localhost:11211 127.0.0.1 600 - -/*/ - -/* eslint-disable no-console */ -var net = require('net'); -var Memcached = require('memcached'); -const P = require('bluebird'); -P.promisifyAll(Memcached.prototype); - -if (process.argv.length < 5) { - var usage = 'Usage: block-ip.js '; - console.error(usage); - process.exit(1); -} - -if (!/.+:\d+/.test(process.argv[2])) { - var x = "use a host:port like 'localhost:11211'"; - console.error(x); - process.exit(1); -} - -if (!net.isIPv4(process.argv[3])) { - console.error('second argument must be an IPv4'); - process.exit(1); -} - -var lifetime = parseInt(process.argv[4], 10); -if (!lifetime || lifetime < 0) { - console.error('third argument must be a positive integer of seconds'); - process.exit(1); -} - -var mc = new Memcached(process.argv[2], { namespace: 'fxa~' }); - -var IpRecord = require('../lib/ip_record')({ - blockIntervalMs: lifetime * 1000, - ipRateLimitIntervalMs: lifetime * 1000, - ipRateLimitBanDurationMs: lifetime * 1000, -}); - -const { fetchRecord, setRecord } = require('../lib/records')( - mc, - {}, - {}, - lifetime -); - -async function main() { - const ip = process.argv[3]; - - try { - const ipRecord = await fetchRecord(ip, IpRecord.parse); - ipRecord.block(); - - await setRecord(ipRecord); - - console.log('successfully blocked', ipRecord, lifetime); - } catch (err) { - console.error(err); - return process.exit(1); - } - - mc.end(); -} - -main(); diff --git a/packages/fxa-customs-server/scripts/config-list.js b/packages/fxa-customs-server/scripts/config-list.js deleted file mode 100755 index 7adde34fbb..0000000000 --- a/packages/fxa-customs-server/scripts/config-list.js +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env node -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/*/ - - Use in combination with config-set.js to update customs server settings - in memcache. config-list.js reads the settings from the memcache server - specified by the first argument and writes json to stdout. - - Example: - - $ config-list.js localhost:11211 > settings.json - $ vi settings.json - $ cat settings.json | config-set.js localhost:11211 - - /*/ - -/* eslint-disable no-console */ -var Memcached = require('memcached'); -var P = require('bluebird'); - -P.promisifyAll(Memcached.prototype); - -if (process.argv.length < 3) { - var usage = 'Usage: config-list.js '; - console.error(usage); - process.exit(1); -} - -if (!/.+:\d+/.test(process.argv[2])) { - var x = "use a host:port like 'localhost:11211'"; - console.error(x); - process.exit(1); -} - -var mc = new Memcached(process.argv[2], { namespace: 'fxa~' }); - -var output = { - limits: {}, - allowedIPs: [], - allowedEmailDomains: [], - allowedPhoneNumbers: [], - requestChecks: {}, -}; - -mc.getAsync('limits') - .then(function (data) { - if (!data) { - console.error('no limits set'); - } else { - output.limits = data; - } - return mc.getAsync('allowedIPs'); - }) - .then(function (data) { - if (!data) { - console.error('no allowedIPs set'); - } else { - output.allowedIPs = data; - } - return mc.getAsync('allowedEmailDomains'); - }) - .then(function (data) { - if (!data) { - console.error('no allowedEmailDomains set'); - } else { - output.allowedEmailDomains = data; - } - return mc.getAsync('allowedPhoneNumbers'); - }) - .then(function (data) { - if (!data) { - console.error('no allowedPhoneNumbers set'); - } else { - output.allowedPhoneNumbers = data; - } - return mc.getAsync('requestChecks'); - }) - .then(function (data) { - if (!data) { - console.error('no requestChecks set'); - } else { - output.requestChecks = data; - } - }) - .then(function () { - mc.end(); - console.log(JSON.stringify(output, null, 2)); - }); diff --git a/packages/fxa-customs-server/scripts/config-set.js b/packages/fxa-customs-server/scripts/config-set.js deleted file mode 100755 index 71efad9fa5..0000000000 --- a/packages/fxa-customs-server/scripts/config-set.js +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env node -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/*/ - - Use in combination with config-list.js to update customs server settings - in memcache. config-set.js reads json from stdin and writes it to the - memcache server specified by the first argument. - - Example: - - $ config-list.js localhost:11211 > settings.json - $ vi settings.json - $ cat settings.json | config-set.js localhost:11211 - - /*/ - -/* eslint-disable no-console */ -var Memcached = require('memcached'); -var P = require('bluebird'); -var BL = require('bl'); -var merge = require('lodash.merge'); -var isEqual = require('lodash.isequal'); - -P.promisifyAll(Memcached.prototype); - -if (process.argv.length < 3) { - var usage = 'Usage: cat settings.json | config-set.js '; - console.error(usage); - process.exit(1); -} - -if (!/.+:\d+/.test(process.argv[2])) { - var x = "use a host:port like 'localhost:11211'"; - console.error(x); - process.exit(1); -} - -function writeMergedSettings(mc, key, newSettings) { - return mc.getAsync(key).then(function (settings) { - if (typeof newSettings !== 'object' || Array.isArray(newSettings)) { - throw new Error('tried to merge non-Object-typed settings value'); - } - settings = settings || {}; - merge(settings, newSettings); - return mc.setAsync(key, settings, 0); - }); -} - -function clobberSettings(mc, key, newSettings) { - return mc.getAsync(key).then(function (settings) { - if (!Array.isArray(newSettings)) { - throw new Error('tried to clobber non-Array-typed settings value'); - } - if (settings && !isEqual(settings, newSettings)) { - console.warn('Clobbering existing settings for "' + key + '":'); - console.warn('Old:', settings); - console.warn('New:', newSettings); - } - return mc.setAsync(key, newSettings, 0); - }); -} - -var b = process.stdin.pipe(new BL()); - -process.stdin.on('end', function () { - var input = null; - try { - input = JSON.parse(b.toString()); - } catch (e) { - return console.error('Input is not valid JSON'); - } - var mc = new Memcached(process.argv[2], { namespace: 'fxa~' }); - - var actions = []; - if (input.limits) { - actions.push(writeMergedSettings(mc, 'limits', input.limits)); - } - - if (input.allowedIPs) { - // It's an array, we can't sensibly merge it. - actions.push(clobberSettings(mc, 'allowedIPs', input.allowedIPs)); - } - - if (input.allowedEmailDomains) { - // It's an array, we can't sensibly merge it. - actions.push( - clobberSettings(mc, 'allowedEmailDomains', input.allowedEmailDomains) - ); - } - - if (input.allowedPhoneNumbers) { - // It's an array, we can't sensibly merge it. - actions.push( - clobberSettings(mc, 'allowedPhoneNumbers', input.allowedPhoneNumbers) - ); - } - - if (input.requestChecks) { - actions.push(writeMergedSettings(mc, 'requestChecks', input.requestChecks)); - } - - P.all(actions) - .catch(function (err) { - console.error('update failed'); - console.error(err); - }) - .then(function () { - mc.end(); - }); -}); diff --git a/packages/fxa-customs-server/scripts/customs-info.js b/packages/fxa-customs-server/scripts/customs-info.js deleted file mode 100755 index e628a7e7a0..0000000000 --- a/packages/fxa-customs-server/scripts/customs-info.js +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env node -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/*/ - -Look up customs info for an ip, email, or ip+email. - -Examples: - -$ ./customs-info.js localhost:11211 127.0.0.1 -$ ./customs-info.js localhost:11211 test@example.com -$ ./customs-info.js localhost:11211 127.0.0.1test@example.com - -/*/ - -/* eslint-disable no-console */ -var Memcached = require('memcached'); - -if (process.argv.length < 4) { - var usage = 'Usage: customs-info.js '; - console.error(usage); - process.exit(1); -} - -if (!/.+:\d+/.test(process.argv[2])) { - var x = "use a host:port like 'localhost:11211'"; - console.error(x); - process.exit(1); -} - -var mc = new Memcached(process.argv[2], { namespace: 'fxa~' }); - -mc.get(process.argv[3], function (err, data) { - if (err) { - console.error(err); - process.exit(1); - } - if (!data) { - mc.end(); - console.error('no record'); - process.exit(1); - } - if (data.bk) { - console.error('blocked at %s', new Date(data.bk)); - } - if (data.lf && data.lf.length > 0) { - console.error('login failures: %d', data.lf.length); - } - if (data.as && data.as.length > 0) { - console.error('account status: %d', data.as.length); - } - if (data.rl) { - console.error('rate limited at %s', new Date(data.rl)); - } - if (data.xs && data.xs.length > 0) { - console.error('emails sent %d', data.xs.length); - } - if (data.pr) { - console.error('password reset at %s', new Date(data.pr)); - } - console.error('raw data:'); - console.log(JSON.stringify(data, null, 2)); - mc.end(); -}); diff --git a/packages/fxa-customs-server/scripts/test-local.sh b/packages/fxa-customs-server/scripts/test-local.sh index 69450d8ae0..e7d78bb95a 100755 --- a/packages/fxa-customs-server/scripts/test-local.sh +++ b/packages/fxa-customs-server/scripts/test-local.sh @@ -4,8 +4,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -echo 'Testing with Memcache database' -tap test/local test/remote test/scripts --no-coverage --jobs=1 - -echo 'Testing with Redis database' -CUSTOMS_REDIS_ENABLED=true tap test/local test/remote test/scripts --no-coverage --jobs=1 +tap test/local test/remote --no-coverage --jobs=1 diff --git a/packages/fxa-customs-server/test/memcache-helper.js b/packages/fxa-customs-server/test/cache-helper.js similarity index 64% rename from packages/fxa-customs-server/test/memcache-helper.js rename to packages/fxa-customs-server/test/cache-helper.js index 859549c267..2e806db8bc 100644 --- a/packages/fxa-customs-server/test/memcache-helper.js +++ b/packages/fxa-customs-server/test/cache-helper.js @@ -8,12 +8,9 @@ var P = require('bluebird'); const Cache = require('../lib/cache'); var config = { - memcache: { - address: process.env.MEMCACHE_ADDRESS || 'localhost:11211', - }, redis: { customs: { - enabled: process.env.CUSTOMS_REDIS_ENABLED === 'true', + enabled: 'true', host: 'localhost', password: '', port: 6379, @@ -87,44 +84,21 @@ var IpRecord = require('../lib/ip_record')(limits); module.exports.limits = limits; async function blockedEmailCheck(email, cb) { - if (config.redis.customs.enabled) { - return P.resolve(true).then(async () => { - const result = await mc.getAsync(email); - var er = EmailRecord.parse(result); - cb(er.shouldBlock()); - }); - } - // give memcache time to flush the writes - setTimeout(function () { - mc.client.get(email, function (err, data) { - var er = EmailRecord.parse(data); - mc.client.end(); - cb(er.shouldBlock()); - }); + return P.resolve(true).then(async () => { + const result = await mc.getAsync(email); + var er = EmailRecord.parse(result); + cb(er.shouldBlock()); }); } module.exports.blockedEmailCheck = blockedEmailCheck; function blockedIpCheck(cb) { - if (config.redis.customs.enabled) { - return Promise.resolve(true).then(async () => { - const result = await mc.getAsync(TEST_IP); - var er = IpRecord.parse(result); - cb(er.shouldBlock()); - }); - } - - setTimeout( - // give memcache time to flush the writes - function () { - mc.get(TEST_IP, function (err, data) { - var ir = IpRecord.parse(data); - mc.end(); - cb(ir.shouldBlock()); - }); - } - ); + return Promise.resolve(true).then(async () => { + const result = await mc.getAsync(TEST_IP); + var er = IpRecord.parse(result); + cb(er.shouldBlock()); + }); } module.exports.blockedIpCheck = blockedIpCheck; @@ -150,60 +124,12 @@ function badLoginCheck() { module.exports.badLoginCheck = badLoginCheck; function clearEverything(cb) { - if (config.redis.customs.enabled) { - return mc.client.redis.flushall(function (err) { - if (err) { - return cb(err); - } - cb(); - }); - } - - mc.client - .itemsAsync() - .then(function (result) { - var firstServer = result[0]; - - // we don't need the "server" key, but the other indicate the slab id's - var keys = Object.keys(firstServer) - .filter((k) => /[0-9]+/i.test(k)) - .map(Number); - - // get a cachedump for each slabid and slab.number - var cachedumps = keys - .map(function (stats) { - return mc.client.cachedumpAsync( - firstServer.server, - stats, - firstServer[stats].number - ); - }) - .map(function (dumpPromise) { - return dumpPromise.then(function (dump) { - if (!dump) { - return; - } - // when one key is return as an object pretend it's an array - if (dump.key && !dump.length) { - dump = [dump]; - } - return P.all( - dump - .filter((item) => /^fxa~/.test(item.key)) - .map(function (item) { - return mc.client.delAsync(item.key.replace(/^fxa~/, '')); - }) - ); - }); - }); - - return P.all(cachedumps); - }) - .then(function () { - mc.client.end(); - cb(); - }) - .catch(cb); + return mc.client.redis.flushall(function (err) { + if (err) { + return cb(err); + } + cb(); + }); } module.exports.clearEverything = clearEverything; @@ -268,6 +194,4 @@ function setRequestChecks(settings) { module.exports.setAllowedEmailDomains = setAllowedEmailDomains; -if (config.redis.customs.enabled) { - mc.client.end = function () {}; -} +mc.client.end = function () {}; diff --git a/packages/fxa-customs-server/test/integration/reputation/docker-compose.yml b/packages/fxa-customs-server/test/integration/reputation/docker-compose.yml index bd5f4c1cf9..9c7b748826 100644 --- a/packages/fxa-customs-server/test/integration/reputation/docker-compose.yml +++ b/packages/fxa-customs-server/test/integration/reputation/docker-compose.yml @@ -19,26 +19,16 @@ services: volumes: - ${PWD}/iprepd.yaml:/app/iprepd.yaml - memcached: - container_name: customs_memcached - image: memcached:1.4 - ports: - - '11211:11211' - customs: container_name: customs_app image: fxa-customs-server:test build: context: ../../.. dockerfile: Dockerfile - environment: - MEMCACHE_ADDRESS: memcached:11211 links: - iprepd:iprepd.host - - memcached depends_on: - iprepd - - memcached ports: - '7000:7000' command: node_modules/.bin/tap test/integration/reputation/iprepd_tests.js diff --git a/packages/fxa-customs-server/test/integration/reputation/iprepd_tests.js b/packages/fxa-customs-server/test/integration/reputation/iprepd_tests.js index b59ecf3d7d..b5c9414e89 100644 --- a/packages/fxa-customs-server/test/integration/reputation/iprepd_tests.js +++ b/packages/fxa-customs-server/test/integration/reputation/iprepd_tests.js @@ -5,7 +5,7 @@ var test = require('tap').test; var TestServer = require('../../test_server'); var Promise = require('bluebird'); var restifyClients = require('restify-clients'); -var mcHelper = require('../../memcache-helper'); +var mcHelper = require('../../cache-helper'); var TEST_EMAIL = 'test@example.com'; var TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/local/settings_tests.js b/packages/fxa-customs-server/test/local/settings_tests.js index 2e5b170451..9dcc897d54 100644 --- a/packages/fxa-customs-server/test/local/settings_tests.js +++ b/packages/fxa-customs-server/test/local/settings_tests.js @@ -74,7 +74,7 @@ test('refresh pushOnMissing works on Missing error', (t) => { }); test('refresh pushOnMissing returns other Errors', (t) => { - const mcError = new Error('memcached error'); + const mcError = new Error('cache error'); mc.getAsync = () => P.reject(mcError); mc.setAsync = (key, val) => { return P.reject(new Error('setAsync should not have been called')); diff --git a/packages/fxa-customs-server/test/remote/block_ip_tests.js b/packages/fxa-customs-server/test/remote/block_ip_tests.js index 96a7fd4ec3..bd50e87fec 100644 --- a/packages/fxa-customs-server/test/remote/block_ip_tests.js +++ b/packages/fxa-customs-server/test/remote/block_ip_tests.js @@ -5,7 +5,7 @@ var test = require('tap').test; var TestServer = require('../test_server'); var Promise = require('bluebird'); var restifyClients = require('restify-clients'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const testUtils = require('../utils'); var TEST_EMAIL = 'test@example.com'; diff --git a/packages/fxa-customs-server/test/remote/check_reputation_tests.js b/packages/fxa-customs-server/test/remote/check_reputation_tests.js index 0958547bec..3467cbecd3 100644 --- a/packages/fxa-customs-server/test/remote/check_reputation_tests.js +++ b/packages/fxa-customs-server/test/remote/check_reputation_tests.js @@ -6,7 +6,7 @@ var TestServer = require('../test_server'); var ReputationServer = require('../test_reputation_server'); var Promise = require('bluebird'); var restifyClients = require('restify-clients'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_EMAIL = 'test@example.com'; var TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/check_tests.js b/packages/fxa-customs-server/test/remote/check_tests.js index 311e6c1b01..e0e3beda73 100644 --- a/packages/fxa-customs-server/test/remote/check_tests.js +++ b/packages/fxa-customs-server/test/remote/check_tests.js @@ -3,7 +3,7 @@ 'use strict'; -const mcHelper = require('../memcache-helper'); +const mcHelper = require('../cache-helper'); const Promise = require('bluebird'); const restifyClients = require('restify-clients'); const test = require('tap').test; @@ -60,17 +60,16 @@ Promise.promisifyAll(client, { multiArgs: true }); }); test('missing action', function (t) { - client.post('/check', { email: TEST_EMAIL, ip: TEST_IP }, function ( - err, - req, - res, - obj - ) { - t.equal(res.statusCode, 400, 'bad request returns a 400'); - t.type(obj.code, 'string', 'bad request returns an error code'); - t.type(obj.message, 'string', 'bad request returns an error message'); - t.end(); - }); + client.post( + '/check', + { email: TEST_EMAIL, ip: TEST_IP }, + function (err, req, res, obj) { + t.equal(res.statusCode, 400, 'bad request returns a 400'); + t.type(obj.code, 'string', 'bad request returns an error code'); + t.type(obj.message, 'string', 'bad request returns an error message'); + t.end(); + } + ); }); test('missing email, ip and action', function (t) { diff --git a/packages/fxa-customs-server/test/remote/config_update_tests.js b/packages/fxa-customs-server/test/remote/config_update_tests.js index dcbda555ea..28a27f50b9 100644 --- a/packages/fxa-customs-server/test/remote/config_update_tests.js +++ b/packages/fxa-customs-server/test/remote/config_update_tests.js @@ -5,7 +5,7 @@ var test = require('tap').test; var TestServer = require('../test_server'); var Promise = require('bluebird'); var restifyClients = Promise.promisifyAll(require('restify-clients')); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomIp, randomEmail, @@ -53,7 +53,7 @@ test('change limits', function (t) { }) .then(function (settings) { t.equal(x + 1, settings.blockIntervalSeconds, 'helper sees the change'); - // Wait for background polling to detect the new value in memcache + // Wait for background polling to detect the new value in cache return Promise.delay(1010); }) .then(function () { @@ -83,7 +83,7 @@ test('change nested limits', function (t) { }) .then(function (settings) { t.equal(x + 1, settings.maxChecksPerUid, 'helper sees the change'); - // Wait for background polling to detect the new value in memcache + // Wait for background polling to detect the new value in cache return Promise.delay(1010); }) .then(function () { @@ -110,7 +110,7 @@ test('change allowedIPs', function (t) { }) .then(function (ips) { t.deepEqual(x, ips, 'helper sees the change'); - // Wait for background polling to detect the new value in memcache + // Wait for background polling to detect the new value in cache return Promise.delay(1010); }) .then(function () { @@ -137,7 +137,7 @@ test('change allowedEmailDomains', function (t) { }) .then(function (ips) { t.deepEqual(x, ips, 'helper sees the change'); - // Wait for background polling to detect the new value in memcache + // Wait for background polling to detect the new value in cache return Promise.delay(1010); }) .then(function () { @@ -168,7 +168,7 @@ test('change allowedPhoneNumbers', function (t) { }) .then(function (phoneNumbers) { t.deepEqual(allowedPhoneNumbers, phoneNumbers, 'helper sees the change'); - // Wait for background polling to detect the new value in memcache + // Wait for background polling to detect the new value in cache return Promise.delay(1010); }) .then(function () { diff --git a/packages/fxa-customs-server/test/remote/email_normalization.js b/packages/fxa-customs-server/test/remote/email_normalization.js index ae293b6f78..c3d3ecebd0 100644 --- a/packages/fxa-customs-server/test/remote/email_normalization.js +++ b/packages/fxa-customs-server/test/remote/email_normalization.js @@ -5,7 +5,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); var Promise = require('bluebird'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_EMAIL = 'test@example.com'; var TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/failed_login_attempt_tests.js b/packages/fxa-customs-server/test/remote/failed_login_attempt_tests.js index 712b8d37b1..b155ea2ba9 100644 --- a/packages/fxa-customs-server/test/remote/failed_login_attempt_tests.js +++ b/packages/fxa-customs-server/test/remote/failed_login_attempt_tests.js @@ -4,7 +4,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomEmail, randomIp } = require('../utils'); var TEST_EMAIL = randomEmail(); diff --git a/packages/fxa-customs-server/test/remote/flow_id_checks.js b/packages/fxa-customs-server/test/remote/flow_id_checks.js index 32e7f69beb..0ca24c7284 100644 --- a/packages/fxa-customs-server/test/remote/flow_id_checks.js +++ b/packages/fxa-customs-server/test/remote/flow_id_checks.js @@ -5,7 +5,7 @@ var test = require('tap').test; var clients = require('restify-clients'); var Promise = require('bluebird'); var TestServer = require('../test_server'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_EMAIL = 'test@example.com'; var TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/ip_blocklist_disable.js b/packages/fxa-customs-server/test/remote/ip_blocklist_disable.js index ef942b86a1..80688f70d9 100644 --- a/packages/fxa-customs-server/test/remote/ip_blocklist_disable.js +++ b/packages/fxa-customs-server/test/remote/ip_blocklist_disable.js @@ -5,7 +5,7 @@ var test = require('tap').test; var Promise = require('bluebird'); var restifyClients = Promise.promisifyAll(require('restify-clients')); var TestServer = require('../test_server'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomEmail } = require('../utils'); var TEST_EMAIL = randomEmail(); diff --git a/packages/fxa-customs-server/test/remote/ip_blocklist_tests.js b/packages/fxa-customs-server/test/remote/ip_blocklist_tests.js index 1292478fe9..b0be5f6276 100644 --- a/packages/fxa-customs-server/test/remote/ip_blocklist_tests.js +++ b/packages/fxa-customs-server/test/remote/ip_blocklist_tests.js @@ -5,7 +5,7 @@ var test = require('tap').test; var Promise = require('bluebird'); var restifyClients = Promise.promisifyAll(require('restify-clients')); var TestServer = require('../test_server'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomEmail } = require('../utils'); var TEST_EMAIL = randomEmail(); diff --git a/packages/fxa-customs-server/test/remote/memcache_error_tests.js b/packages/fxa-customs-server/test/remote/memcache_error_tests.js deleted file mode 100644 index a5d45dc76e..0000000000 --- a/packages/fxa-customs-server/test/remote/memcache_error_tests.js +++ /dev/null @@ -1,55 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -var test = require('tap').test; -var restifyClients = require('restify-clients'); -var TestServer = require('../test_server'); -var mcHelper = require('../memcache-helper'); - -var TEST_EMAIL = 'test@example.com'; -var TEST_IP = '192.0.2.1'; - -const config = require('../../lib/config').getProperties(); -config.memcache.address = '128.0.0.1:12131'; -config.redis.customs.port = '6380'; - -var testServer = new TestServer(config); - -test('startup', async function (t) { - await testServer.start(); - t.type(testServer.server, 'object', 'test server was started'); - t.end(); -}); - -test('clear everything', function (t) { - mcHelper.clearEverything(function (err) { - t.notOk(err, 'no errors were returned'); - t.end(); - }); -}); - -var client = restifyClients.createJsonClient({ - url: 'http://localhost:' + config.listen.port, -}); - -test( - 'request with disconnected memcache', - { skip: config.redis.customs.enabled }, - function (t) { - client.post( - '/check', - { email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' }, - function (err, req, res, obj) { - t.equal(res.statusCode, 200, 'check worked'); - t.equal(obj.block, true, 'request was blocked'); - t.equal(obj.retryAfter, 900, 'retry after'); - t.end(); - } - ); - } -); - -test('teardown', async function (t) { - await testServer.stop(); - t.end(); -}); diff --git a/packages/fxa-customs-server/test/remote/never_blocked.js b/packages/fxa-customs-server/test/remote/never_blocked.js index 985557f4ef..f4a786cc5a 100644 --- a/packages/fxa-customs-server/test/remote/never_blocked.js +++ b/packages/fxa-customs-server/test/remote/never_blocked.js @@ -5,7 +5,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); var Promise = require('bluebird'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_EMAIL = 'test@example.com'; var TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/password_reset_clears_bad_logins.js b/packages/fxa-customs-server/test/remote/password_reset_clears_bad_logins.js index 6059385b3f..ada0aca4b8 100644 --- a/packages/fxa-customs-server/test/remote/password_reset_clears_bad_logins.js +++ b/packages/fxa-customs-server/test/remote/password_reset_clears_bad_logins.js @@ -5,7 +5,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); var Promise = require('bluebird'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomEmail, randomIp } = require('../utils'); var TEST_EMAIL = randomEmail(); diff --git a/packages/fxa-customs-server/test/remote/password_reset_tests.js b/packages/fxa-customs-server/test/remote/password_reset_tests.js index 36faae4388..e6c56cc190 100644 --- a/packages/fxa-customs-server/test/remote/password_reset_tests.js +++ b/packages/fxa-customs-server/test/remote/password_reset_tests.js @@ -4,7 +4,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomEmail } = require('../utils'); var TEST_EMAIL = randomEmail(); diff --git a/packages/fxa-customs-server/test/remote/send_violations_tests.js b/packages/fxa-customs-server/test/remote/send_violations_tests.js index 0f12bdda09..53bac19507 100644 --- a/packages/fxa-customs-server/test/remote/send_violations_tests.js +++ b/packages/fxa-customs-server/test/remote/send_violations_tests.js @@ -7,7 +7,7 @@ var TestServer = require('../test_server'); var ReputationServer = require('../test_reputation_server'); var Promise = require('bluebird'); var restifyClients = require('restify-clients'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const testUtils = require('../utils'); var TEST_EMAIL = 'test@example.com'; diff --git a/packages/fxa-customs-server/test/remote/too_many_account_status_check.js b/packages/fxa-customs-server/test/remote/too_many_account_status_check.js index 1e0c3606e9..f3bb704441 100644 --- a/packages/fxa-customs-server/test/remote/too_many_account_status_check.js +++ b/packages/fxa-customs-server/test/remote/too_many_account_status_check.js @@ -5,7 +5,7 @@ var test = require('tap').test; var TestServer = require('../test_server'); var Promise = require('bluebird'); var restifyClients = Promise.promisifyAll(require('restify-clients')); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_IP = '192.0.2.1'; var ACCOUNT_STATUS_CHECK = 'accountStatusCheck'; diff --git a/packages/fxa-customs-server/test/remote/too_many_authenticated_checks.js b/packages/fxa-customs-server/test/remote/too_many_authenticated_checks.js index bea722423e..0cbb2ab28f 100644 --- a/packages/fxa-customs-server/test/remote/too_many_authenticated_checks.js +++ b/packages/fxa-customs-server/test/remote/too_many_authenticated_checks.js @@ -5,7 +5,7 @@ var test = require('tap').test; var TestServer = require('../test_server'); var Promise = require('bluebird'); var restifyClients = Promise.promisifyAll(require('restify-clients')); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_IP = '192.0.2.1'; var TEST_UID = 'abc123'; diff --git a/packages/fxa-customs-server/test/remote/too_many_bad_logins.js b/packages/fxa-customs-server/test/remote/too_many_bad_logins.js index a5d69032c3..abdb3b1ecd 100644 --- a/packages/fxa-customs-server/test/remote/too_many_bad_logins.js +++ b/packages/fxa-customs-server/test/remote/too_many_bad_logins.js @@ -5,7 +5,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); var Promise = require('bluebird'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_EMAIL = 'test@example.com'; var TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/too_many_emails.js b/packages/fxa-customs-server/test/remote/too_many_emails.js index f8a205dfd9..27a9c81cee 100644 --- a/packages/fxa-customs-server/test/remote/too_many_emails.js +++ b/packages/fxa-customs-server/test/remote/too_many_emails.js @@ -5,7 +5,7 @@ var test = require('tap').test; var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); var Promise = require('bluebird'); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); const { randomEmail, randomIp } = require('../utils'); var TEST_EMAIL = randomEmail(); diff --git a/packages/fxa-customs-server/test/remote/too_many_login_checks.js b/packages/fxa-customs-server/test/remote/too_many_login_checks.js index a98c8d3161..1aa06d3450 100644 --- a/packages/fxa-customs-server/test/remote/too_many_login_checks.js +++ b/packages/fxa-customs-server/test/remote/too_many_login_checks.js @@ -10,7 +10,7 @@ var test = require('tap').test; var TestServer = require('../test_server'); var Promise = require('bluebird'); var restifyClients = Promise.promisifyAll(require('restify-clients')); -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var TEST_IP = '192.0.2.1'; var ACCOUNT_LOGIN = 'accountLogin'; diff --git a/packages/fxa-customs-server/test/remote/too_many_signin_codes.js b/packages/fxa-customs-server/test/remote/too_many_signin_codes.js index ae103ccb33..8e9b8da1ff 100644 --- a/packages/fxa-customs-server/test/remote/too_many_signin_codes.js +++ b/packages/fxa-customs-server/test/remote/too_many_signin_codes.js @@ -3,7 +3,7 @@ 'use strict'; -const memcached = require('../memcache-helper'); +const cached = require('../cache-helper'); const Promise = require('bluebird'); const restifyClients = Promise.promisifyAll(require('restify-clients')); const test = require('tap').test; @@ -32,8 +32,8 @@ test('startup', async function (t) { }); test('clear everything', (t) => { - memcached.clearEverything((err) => { - t.notOk(err, 'memcached.clearEverything should not return an error'); + cached.clearEverything((err) => { + t.notOk(err, 'cache.clearEverything should not return an error'); t.end(); }); }); diff --git a/packages/fxa-customs-server/test/remote/too_many_sms.js b/packages/fxa-customs-server/test/remote/too_many_sms.js index dcb959a005..1ebcaecac7 100644 --- a/packages/fxa-customs-server/test/remote/too_many_sms.js +++ b/packages/fxa-customs-server/test/remote/too_many_sms.js @@ -25,7 +25,7 @@ config.limits.ipRateLimitBanDurationSeconds = 1; config.limits.rateLimitIntervalSeconds = 1; config.allowedPhoneNumbers = [ALLOWED_PHONE_NUMBER]; -var mcHelper = require('../memcache-helper'); +var mcHelper = require('../cache-helper'); var testServer = new TestServer(config); diff --git a/packages/fxa-customs-server/test/remote/too_many_verify_codes.js b/packages/fxa-customs-server/test/remote/too_many_verify_codes.js index 1effa7a50f..d197e4db47 100644 --- a/packages/fxa-customs-server/test/remote/too_many_verify_codes.js +++ b/packages/fxa-customs-server/test/remote/too_many_verify_codes.js @@ -5,7 +5,7 @@ const test = require('tap').test; const TestServer = require('../test_server'); const Promise = require('bluebird'); const restifyClients = Promise.promisifyAll(require('restify-clients')); -const mcHelper = require('../memcache-helper'); +const mcHelper = require('../cache-helper'); const testUtils = require('../utils'); const TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/unblock_tests.js b/packages/fxa-customs-server/test/remote/unblock_tests.js index 1bdf314bb0..b8e8e19f86 100644 --- a/packages/fxa-customs-server/test/remote/unblock_tests.js +++ b/packages/fxa-customs-server/test/remote/unblock_tests.js @@ -7,7 +7,7 @@ const test = require('tap').test; const restifyClients = require('restify-clients'); const TestServer = require('../test_server'); const Promise = require('bluebird'); -const mcHelper = require('../memcache-helper'); +const mcHelper = require('../cache-helper'); const TEST_EMAIL = 'test@example.com'; const TEST_IP = '192.0.2.1'; diff --git a/packages/fxa-customs-server/test/remote/user_defined_rules.js b/packages/fxa-customs-server/test/remote/user_defined_rules.js index 17fa452947..5f04d364bc 100644 --- a/packages/fxa-customs-server/test/remote/user_defined_rules.js +++ b/packages/fxa-customs-server/test/remote/user_defined_rules.js @@ -5,7 +5,7 @@ const test = require('tap').test; const TestServer = require('../test_server'); const Promise = require('bluebird'); const restifyClients = Promise.promisifyAll(require('restify-clients')); -const mcHelper = require('../memcache-helper'); +const mcHelper = require('../cache-helper'); function randomEmail() { return Math.floor(Math.random() * 10000) + '@email.com'; diff --git a/packages/fxa-customs-server/test/scripts/block-ip.js b/packages/fxa-customs-server/test/scripts/block-ip.js deleted file mode 100644 index 7518436322..0000000000 --- a/packages/fxa-customs-server/test/scripts/block-ip.js +++ /dev/null @@ -1,18 +0,0 @@ -const ROOT_DIR = '../..'; - -const test = require('tap').test; - -const cp = require('child_process'); -const Promise = require('bluebird'); -const path = require('path'); - -cp.execAsync = Promise.promisify(cp.exec); - -const cwd = path.resolve(__dirname, ROOT_DIR); - -test('can run script', async function (t) { - cp.execAsync('node scripts/block-ip.js localhost:11211 127.0.0.1 600', { - cwd, - }); - t.end(); -}); diff --git a/yarn.lock b/yarn.lock index cf44ea57a0..f8b423b442 100644 --- a/yarn.lock +++ b/yarn.lock @@ -30246,13 +30246,6 @@ __metadata: languageName: node linkType: hard -"connection-parse@npm:0.0.x": - version: 0.0.7 - resolution: "connection-parse@npm:0.0.7" - checksum: 82c242157205babf80bd6467a65b4f05c21eed91cc46f76ca26398ef10b031a7a0bb63efbda3386e6d24dbfaf74a9d786b0e2fb6425b3a159c762b18052a5070 - languageName: node - linkType: hard - "consola@npm:^2.15.0": version: 2.15.0 resolution: "consola@npm:2.15.0" @@ -37524,7 +37517,6 @@ fsevents@~2.1.1: "@types/jws": ^3.2.3 "@types/lodash": ^4 "@types/luxon": ^3 - "@types/memcached": ^2.2.6 "@types/mjml": ^4.7.4 "@types/mocha": ^10.0.6 "@types/nock": ^11.1.0 @@ -37855,7 +37847,6 @@ fsevents@~2.1.1: load-grunt-tasks: ^5.1.0 lodash.isequal: 4.5.0 lodash.merge: 4.6.2 - memcached: ^2.2.2 pm2: ^5.3.0 prettier: ^2.3.1 proxyquire: ^2.1.3 @@ -40644,16 +40635,6 @@ fsevents@~2.1.1: languageName: node linkType: hard -"hashring@npm:3.2.x": - version: 3.2.0 - resolution: "hashring@npm:3.2.0" - dependencies: - connection-parse: 0.0.x - simple-lru-cache: 0.0.x - checksum: d46733d341227a48df48c9c4d31f36a174e8c773df2f9013baed42357f93e07cd99e60dcb40c986b813654d505e5938f0a677bf33935e7be8ee20a0a1fbeb1db - languageName: node - linkType: hard - "hasown@npm:^2.0.0": version: 2.0.0 resolution: "hasown@npm:2.0.0" @@ -43557,15 +43538,6 @@ fsevents@~2.1.1: languageName: node linkType: hard -"jackpot@npm:>=0.0.6": - version: 0.0.6 - resolution: "jackpot@npm:0.0.6" - dependencies: - retry: 0.6.0 - checksum: 57af0f39f8ffbf12825aaabe60295d2bb9c0e56ef6123168a542f73ce56935498ef35350b53171111214f3bcfe1ac4ff75cf2b64662e714adb1218b5e92ad6b9 - languageName: node - linkType: hard - "jackspeak@npm:^1.4.1": version: 1.4.1 resolution: "jackspeak@npm:1.4.1" @@ -48254,16 +48226,6 @@ fsevents@~2.1.1: languageName: node linkType: hard -"memcached@npm:^2.2.2": - version: 2.2.2 - resolution: "memcached@npm:2.2.2" - dependencies: - hashring: 3.2.x - jackpot: ">=0.0.6" - checksum: bb87f0c9422e5a4dc1ca2235115b88ac1b4f3cf0d10435bb1296f54e3c14971ad764b1c26a071c0e24baa6e3095eef8e18ccfe82d70ca8272334ccd4340db1a5 - languageName: node - linkType: hard - "memfs@npm:^3.1.2": version: 3.2.2 resolution: "memfs@npm:3.2.2" @@ -58055,13 +58017,6 @@ resolve@1.1.7: languageName: node linkType: hard -"retry@npm:0.6.0": - version: 0.6.0 - resolution: "retry@npm:0.6.0" - checksum: 44c7b1179ef9627d3ff11b9948e49094226c28f8838c43e39e3cd6a15689daacd59e82db53c5177793717768a6a15e16db6606b5b51cce7622751349c6a1ccfa - languageName: node - linkType: hard - "reusify@npm:^1.0.4": version: 1.0.4 resolution: "reusify@npm:1.0.4" @@ -59523,13 +59478,6 @@ resolve@1.1.7: languageName: node linkType: hard -"simple-lru-cache@npm:0.0.x": - version: 0.0.2 - resolution: "simple-lru-cache@npm:0.0.2" - checksum: 75bc9b788be4dad48b96ff2225604fe9c62d2e69327491ecb4d034b482cb88e5cd810e7b84f6ded3c4a87f1ecd407f1f2fafd5da09cf07dec3b9ce100cc13a44 - languageName: node - linkType: hard - "simple-swizzle@npm:^0.2.2": version: 0.2.2 resolution: "simple-swizzle@npm:0.2.2"