[WT-285]: Feat/Automate removal of expired Send files

bin/expired-send-links/Dockerfile

@@ -0,0 +1,15 @@
+FROM mhart/alpine-node:16
+LABEL author="internxt"
+
+WORKDIR /
+
+# Add useful packages
+RUN apk add git curl
+
+COPY . .
+
+# Install deps
+RUN yarn
+
+# Run command
+CMD ts-node ./bin/expired-send-links/index.ts

bin/expired-send-links/index.ts

@@ -0,0 +1,172 @@
+import { Command } from 'commander';
+import Database from '../../src/config/initializers/database';
+import { UserModel } from '../../src/modules/user/user.model';
+import { FolderModel } from '../../src/modules/folder/folder.model';
+import { SendLinkItemModel } from '../../src/modules/send/send-link-item.model';
+import { DeletedFileModel } from '../../src/modules/deleted-file/deleted-file.model';
+import { SendLinkModel } from '../../src/modules/send/send-link.model';
+import { SendLinkAttributes } from '../../src/modules/send/send-link.domain';
+import { ModelType } from 'sequelize-typescript';
+import {
+  createTimer,
+  getExpiredSendLinks,
+  moveItemsToDeletedFiles,
+  clearExpiredSendLink,
+  clearExpiredSendLinkItems,
+} from './utils';
+
+const commands: { flags: string; description: string }[] = [
+  {
+    flags: '--db-hostname <database_hostname>',
+    description: 'The database hostname',
+  },
+  {
+    flags: '--db-name <database_name>',
+    description: 'The database name',
+  },
+  {
+    flags: '--db-username <database_username>',
+    description:
+      'The username authorized to create from deleted_files table and read/delete from send_links and send_link_items',
+  },
+  {
+    flags: '--db-password <database_password>',
+    description: 'The database username password',
+  },
+  {
+    flags: '--db-port <database_port>',
+    description: 'The database port',
+  },
+  {
+    flags: '--user-id <user_id>',
+    description: 'The user owner of the files',
+  },
+  {
+    flags: '--folder-id <folder_id>',
+    description: 'The folder id where files are stored',
+  },
+  {
+    flags: '--bucket-id <bucket_id>',
+    description: 'The bucket id where the files are stored in the network',
+  },
+  {
+    flags: '-l, --limit [limit]',
+    description: 'The files limit to handle each time',
+  },
+];
+
+const command = new Command('expired-send-links').version('0.0.1');
+
+commands.forEach((c) => {
+  command.option(c.flags, c.description);
+});
+
+command.parse(process.argv);
+
+const opts = command.opts();
+const db = Database.getInstance({
+  sequelizeConfig: {
+    host: opts.dbHostname,
+    port: opts.dbPort,
+    database: opts.dbName,
+    username: opts.dbUsername,
+    password: opts.dbPassword,
+    dialect: 'postgres',
+    repositoryMode: true,
+    dialectOptions: {
+      ssl: {
+        require: true,
+        rejectUnauthorized: false,
+      },
+    },
+  },
+});
+db.addModels([
+  SendLinkModel,
+  UserModel,
+  SendLinkItemModel,
+  FolderModel,
+  DeletedFileModel,
+]);
+
+const timer = createTimer();
+timer.start();
+
+let totalMovedExpiredLinks = 0;
+
+const logIntervalId = setInterval(() => {
+  console.log(
+    'EXPIRED LINKS DELETED RATE: %s/s',
+    totalMovedExpiredLinks / (timer.end() / 1000),
+  );
+}, 10000);
+
+function finishProgram() {
+  clearInterval(logIntervalId);
+
+  console.log(
+    'TOTAL EXPIRED LINKS DELETED %s | DURATION %ss',
+    totalMovedExpiredLinks,
+    (timer.end() / 1000).toFixed(2),
+  );
+  db.close()
+    .then(() => {
+      console.log('DISCONNECTED FROM DB');
+    })
+    .catch((err) => {
+      console.log(
+        'Error closing connection %s. %s',
+        err.message.err.stack || 'NO STACK.',
+      );
+    });
+}
+
+process.on('SIGINT', () => finishProgram());
+
+async function start(limit = 20) {
+  const SendLinkRepository = db.getRepository(SendLinkModel);
+  const SendLinkItemRepository = db.getRepository(SendLinkItemModel);
+  const DeletedFilesRepository = db.getRepository(DeletedFileModel);
+  const sendLinkItemModel = db.models.SendLinkItemModel as unknown as ModelType<
+    SendLinkItemModel,
+    SendLinkItemModel
+  >;
+
+  let expiredLinks: SendLinkAttributes[] = [];
+
+  do {
+    expiredLinks = await getExpiredSendLinks(
+      SendLinkRepository,
+      sendLinkItemModel,
+      limit,
+    );
+
+    console.time('move-to-deleted-files');
+
+    for (const expiredLink of expiredLinks) {
+      for (let i = 0; i < expiredLink.items.length; i += 20) {
+        await moveItemsToDeletedFiles(
+          DeletedFilesRepository,
+          expiredLink.items.slice(i, i + 20),
+          Number(opts.userId),
+          Number(opts.folderId),
+          String(opts.bucketId),
+        );
+      }
+      await clearExpiredSendLinkItems(SendLinkItemRepository, expiredLink);
+      await clearExpiredSendLink(SendLinkRepository, expiredLink);
+    }
+
+    console.timeEnd('move-to-deleted-files');
+
+    totalMovedExpiredLinks += expiredLinks.length;
+  } while (expiredLinks.length === limit);
+}
+
+start(parseInt(opts.limit || '20'))
+  .catch((err) => {
+    console.log('err', err);
+  })
+  .finally(() => {
+    finishProgram();
+  });

bin/expired-send-links/utils.ts

@@ -0,0 +1,111 @@
+import { Op } from 'sequelize';
+import { ModelType, Repository, Sequelize } from 'sequelize-typescript';
+import { DeletedFileModel } from 'src/modules/deleted-file/deleted-file.model';
+import { SendLinkItemAttributes } from '../../src/modules/send/send-link-item.domain';
+import { SendLinkItemModel } from '../../src/modules/send/send-link-item.model';
+import { SendLinkAttributes } from '../../src/modules/send/send-link.domain';
+import { SendLinkModel } from '../../src/modules/send/send-link.model';
+
+type Timer = { start: () => void; end: () => number };
+
+export const createTimer = (): Timer => {
+  let timeStart: [number, number];
+
+  return {
+    start: () => {
+      timeStart = process.hrtime();
+    },
+    end: () => {
+      const NS_PER_SEC = 1e9;
+      const NS_TO_MS = 1e6;
+      const diff = process.hrtime(timeStart);
+
+      return (diff[0] * NS_PER_SEC + diff[1]) / NS_TO_MS;
+    },
+  };
+};
+
+export function getExpiredSendLinks(
+  SendLinkRepository: Repository<SendLinkModel>,
+  SendLinkItemModel: ModelType<SendLinkItemModel, SendLinkItemModel>,
+  limit: number,
+): Promise<SendLinkAttributes[]> {
+  return SendLinkRepository.findAll({
+    limit,
+    order: [['id', 'ASC']],
+    where: {
+      expirationAt: {
+        [Op.lt]: Sequelize.literal('NOW()'),
+      },
+    },
+    include: {
+      model: SendLinkItemModel,
+      attributes: ['id', 'networkId'],
+      where: {
+        networkId: {
+          [Op.not]: null,
+        },
+      },
+    },
+  }).then((res) => {
+    return res as unknown as SendLinkAttributes[];
+  });
+}
+
+export function moveItemsToDeletedFiles(
+  deletedFilesRepository: Repository<DeletedFileModel>,
+  expiredSendLinkItems: SendLinkItemAttributes[],
+  userId: number,
+  folderId: number,
+  bucketId: string,
+): Promise<DeletedFileModel[]> {
+  const deletedFiles: {
+    file_id: string;
+    user_id: number;
+    folder_id: number;
+    bucket: string;
+  }[] = expiredSendLinkItems.map((item) => {
+    return {
+      file_id: item.networkId,
+      user_id: userId,
+      folder_id: folderId,
+      bucket: bucketId,
+    };
+  });
+
+
+  return deletedFilesRepository.bulkCreate(deletedFiles);
+}
+
+export function moveToDeletedFiles(
+  DeletedFilesRepository: Repository<DeletedFileModel>,
+  expiredSendLink: SendLinkItemAttributes,
+  SEND_USER_ID: number,
+  SEND_FOLDER_ID: number,
+  SEND_BUCKET: string,
+): Promise<DeletedFileModel> {
+  return DeletedFilesRepository.create({
+    file_id: expiredSendLink.networkId,
+    user_id: SEND_USER_ID,
+    folder_id: SEND_FOLDER_ID,
+    bucket: SEND_BUCKET,
+  });
+}
+
+export function clearExpiredSendLinkItems(
+  SendLinkItemRepository: Repository<SendLinkItemModel>,
+  expiredLink: SendLinkAttributes,
+) {
+  return SendLinkItemRepository.destroy({
+    where: { link_id: expiredLink.id },
+  });
+}
+
+export function clearExpiredSendLink(
+  SendLinkRepository: Repository<SendLinkModel>,
+  expiredLink: SendLinkAttributes,
+) {
+  return SendLinkRepository.destroy({
+    where: { id: expiredLink.id },
+  });
+}

package.json

@@ -92,6 +92,7 @@
     "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
+    "commander": "^10.0.0",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",

src/config/initializers/database.ts

@@ -0,0 +1,63 @@
+import { Sequelize, SequelizeOptions } from 'sequelize-typescript';
+import { WinstonLogger } from '../../../src/lib/winston-logger';
+import { format } from 'sql-formatter';
+import _ from 'lodash';
+
+export default class Database {
+  private static instance: Sequelize;
+
+  static getInstance(config: any): Sequelize {
+    if (Database.instance) {
+      return Database.instance;
+    }
+
+    const logger = WinstonLogger.getLogger();
+
+    const defaultSettings = {
+      resetAfterUse: true,
+      operatorsAliases: 0,
+      pool: {
+        maxConnections: Number.MAX_SAFE_INTEGER,
+        maxIdleTime: 30000,
+        max: 20,
+        min: 0,
+        idle: 20000,
+        acquire: 20000,
+      },
+      logging: (content: string) => {
+        const parse = content.match(/^(Executing \(.*\):) (.*)$/);
+        if (parse) {
+          const prettySql = format(parse[2]);
+          logger.debug(`${parse[1]}\n${prettySql}`);
+        } else {
+          logger.debug(`Could not parse sql content: ${content}`);
+        }
+      },
+    };
+
+    const sequelizeSettings: SequelizeOptions = _.merge(
+      defaultSettings,
+      config.sequelizeConfig,
+    );
+
+    const instance = new Sequelize(
+      config.name,
+      config.user,
+      config.password,
+      sequelizeSettings,
+    );
+
+    instance
+      .authenticate()
+      .then(() => {
+        logger.log('Connected to database');
+      })
+      .catch((err) => {
+        logger.error('Database connection error: %s', err);
+      });
+
+    Database.instance = instance;
+
+    return instance;
+  }
+}

src/lib/winston-logger.ts

@@ -1,6 +1,7 @@
 import { WinstonModule } from 'nest-winston';
 import winston from 'winston';
 import os from 'os';
+import { LoggerService } from '@nestjs/common';
 
 const { splat, combine, printf, timestamp } = winston.format;
 const serverHostname = os.hostname();
@@ -46,12 +47,12 @@ export class WinstonLogger {
     transports: [new winston.transports.Console()],
   };
 
-  static getLogger() {
+  static getLogger(): LoggerService {
     const configs = {
       production: WinstonModule.createLogger(this.prodOptions),
       staging: WinstonModule.createLogger(this.prodOptions),
-      development: null,
+      development: WinstonModule.createLogger(this.devOptions),
     };
-    return configs[process.env.NODE_EV] || null;
+    return configs[process.env.NODE_EV] || configs.development;
   }
 }