From 72dd698a391eb979501bdd7230ebe19d562515af Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 13:36:26 +0100 Subject: [PATCH 01/12] added database iniciator class --- src/config/initializers/database.ts | 63 +++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/config/initializers/database.ts diff --git a/src/config/initializers/database.ts b/src/config/initializers/database.ts new file mode 100644 index 000000000..b87b82efe --- /dev/null +++ b/src/config/initializers/database.ts @@ -0,0 +1,63 @@ +import { Sequelize, SequelizeOptions } from 'sequelize-typescript'; +import { WinstonLogger } from '../../../src/lib/winston-logger'; +import { format } from 'sql-formatter'; +import _ from 'lodash'; + +export default class Database { + private static instance: Sequelize; + + static getInstance(config: any): Sequelize { + if (Database.instance) { + return Database.instance; + } + + const logger = WinstonLogger.getLogger(); + + const defaultSettings = { + resetAfterUse: true, + operatorsAliases: 0, + pool: { + maxConnections: Number.MAX_SAFE_INTEGER, + maxIdleTime: 30000, + max: 20, + min: 0, + idle: 20000, + acquire: 20000, + }, + logging: (content: string) => { + const parse = content.match(/^(Executing \(.*\):) (.*)$/); + if (parse) { + const prettySql = format(parse[2]); + logger.debug(`${parse[1]}\n${prettySql}`); + } else { + logger.debug(`Could not parse sql content: ${content}`); + } + }, + }; + + const sequelizeSettings: SequelizeOptions = _.merge( + defaultSettings, + config.sequelizeConfig, + ); + + const instance = new Sequelize( + config.name, + config.user, + config.password, + sequelizeSettings, + ); + + instance + .authenticate() + .then(() => { + logger.info('Connected to database'); + }) + .catch((err) => { + logger.error('Database connection error: %s', err); + }); + + Database.instance = instance; + + return instance; + } +} From 88ab68814352584d71b90dd32487d19f051b9292 Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 13:41:44 +0100 Subject: [PATCH 02/12] added deleted file model --- .../deleted-file/deleted-file.domain.ts | 38 +++++++++++++++++++ .../deleted-file/deleted-file.model.ts | 22 +++++++++++ .../deleted-file/dto/deleted-file.dto.ts | 6 +++ 3 files changed, 66 insertions(+) create mode 100644 src/modules/deleted-file/deleted-file.domain.ts create mode 100644 src/modules/deleted-file/deleted-file.model.ts create mode 100644 src/modules/deleted-file/dto/deleted-file.dto.ts diff --git a/src/modules/deleted-file/deleted-file.domain.ts b/src/modules/deleted-file/deleted-file.domain.ts new file mode 100644 index 000000000..04d9c0e1a --- /dev/null +++ b/src/modules/deleted-file/deleted-file.domain.ts @@ -0,0 +1,38 @@ +import { DeletedFileDto } from './dto/deleted-file.dto'; +export interface DeletedFileAttributes { + file_id: string; + user_id: number; + folder_id: number; + bucket: string; +} + +export class DeletedFile implements DeletedFileAttributes { + file_id: string; + user_id: number; + folder_id: number; + bucket: string; + private constructor({ + file_id, + user_id, + folder_id, + bucket, + }: DeletedFileAttributes) { + this.file_id = file_id; + this.user_id = user_id; + this.folder_id = folder_id; + this.bucket = bucket; + } + + static build(file: DeletedFileAttributes): DeletedFile { + return new DeletedFile(file); + } + + toJSON(): DeletedFileDto { + return { + file_id: this.file_id, + user_id: this.user_id, + folder_id: this.folder_id, + bucket: this.bucket, + }; + } +} diff --git a/src/modules/deleted-file/deleted-file.model.ts b/src/modules/deleted-file/deleted-file.model.ts new file mode 100644 index 000000000..259fb5f88 --- /dev/null +++ b/src/modules/deleted-file/deleted-file.model.ts @@ -0,0 +1,22 @@ +import { Table, Model, PrimaryKey, Column } from 'sequelize-typescript'; +import { DeletedFileAttributes } from './deleted-file.domain'; + +@Table({ + underscored: true, + timestamps: false, + tableName: 'deleted_files', +}) +export class DeletedFileModel extends Model implements DeletedFileAttributes { + @PrimaryKey + @Column + file_id: string; + + @Column + user_id: number; + + @Column + folder_id: number; + + @Column + bucket: string; +} diff --git a/src/modules/deleted-file/dto/deleted-file.dto.ts b/src/modules/deleted-file/dto/deleted-file.dto.ts new file mode 100644 index 000000000..09e61a27f --- /dev/null +++ b/src/modules/deleted-file/dto/deleted-file.dto.ts @@ -0,0 +1,6 @@ +export class DeletedFileDto { + file_id: string; + user_id: number; + folder_id: number; + bucket: string; +} From 7fcd356af65c519c04e48f4bc31f5f382cef44a9 Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 14:54:14 +0100 Subject: [PATCH 03/12] moved folder model --- src/modules/file/file.repository.ts | 2 +- src/modules/folder/folder.model.ts | 65 ++++++++++++++++++++++++ src/modules/folder/folder.module.ts | 3 +- src/modules/folder/folder.repository.ts | 66 +------------------------ src/modules/send/send.module.ts | 2 +- src/modules/send/send.usecase.spec.ts | 2 +- src/modules/share/share.module.ts | 6 +-- src/modules/share/share.repository.ts | 2 +- 8 files changed, 75 insertions(+), 73 deletions(-) create mode 100644 src/modules/folder/folder.model.ts diff --git a/src/modules/file/file.repository.ts b/src/modules/file/file.repository.ts index 5370e971a..6f12c7810 100644 --- a/src/modules/file/file.repository.ts +++ b/src/modules/file/file.repository.ts @@ -2,7 +2,7 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { InjectModel } from '@nestjs/sequelize'; import { File, FileAttributes, FileOptions } from './file.domain'; import sequelize, { FindOptions, Op } from 'sequelize'; -import { FolderModel } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { AllowNull, diff --git a/src/modules/folder/folder.model.ts b/src/modules/folder/folder.model.ts new file mode 100644 index 000000000..74128e56b --- /dev/null +++ b/src/modules/folder/folder.model.ts @@ -0,0 +1,65 @@ +import { + Table, + Model, + PrimaryKey, + AutoIncrement, + Column, + ForeignKey, + BelongsTo, + Index, + DataType, + Default, + AllowNull, +} from 'sequelize-typescript'; +import { UserModel } from '../user/user.model'; +import { FolderAttributes } from './folder.domain'; + +@Table({ + underscored: true, + timestamps: true, + tableName: 'folders', +}) +export class FolderModel extends Model implements FolderAttributes { + @PrimaryKey + @AutoIncrement + @Column + id: number; + + @ForeignKey(() => FolderModel) + @Column + parentId: number; + + @BelongsTo(() => FolderModel) + parent: FolderModel; + + @Index + @Column + name: string; + + @Column(DataType.STRING(24)) + bucket: string; + + @ForeignKey(() => UserModel) + @Column + userId: number; + + @BelongsTo(() => UserModel) + user: UserModel; + + @Column + encryptVersion: '03-aes'; + + @Default(false) + @Column + deleted: boolean; + + @AllowNull + @Column + deletedAt: Date; + + @Column + createdAt: Date; + + @Column + updatedAt: Date; +} diff --git a/src/modules/folder/folder.module.ts b/src/modules/folder/folder.module.ts index 4d8194e49..d6ffd95be 100644 --- a/src/modules/folder/folder.module.ts +++ b/src/modules/folder/folder.module.ts @@ -1,6 +1,7 @@ import { Module, forwardRef } from '@nestjs/common'; import { SequelizeModule } from '@nestjs/sequelize'; -import { FolderModel, SequelizeFolderRepository } from './folder.repository'; +import { SequelizeFolderRepository } from './folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { FolderUseCases } from './folder.usecase'; import { FileModule } from '../file/file.module'; import { CryptoModule } from '../../externals/crypto/crypto.module'; diff --git a/src/modules/folder/folder.repository.ts b/src/modules/folder/folder.repository.ts index 249fd49ba..f63ef8061 100644 --- a/src/modules/folder/folder.repository.ts +++ b/src/modules/folder/folder.repository.ts @@ -2,72 +2,10 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { InjectModel } from '@nestjs/sequelize'; import { Folder, FolderAttributes } from './folder.domain'; import { FindOptions, Op } from 'sequelize'; -import { - AllowNull, - AutoIncrement, - BelongsTo, - Column, - DataType, - Default, - ForeignKey, - Index, - Model, - PrimaryKey, - Table, -} from 'sequelize-typescript'; -import { UserModel } from '../user/user.repository'; import { User, UserAttributes } from '../user/user.domain'; import { Pagination } from '../../lib/pagination'; - -@Table({ - underscored: true, - timestamps: true, - tableName: 'folders', -}) -export class FolderModel extends Model implements FolderAttributes { - @PrimaryKey - @AutoIncrement - @Column - id: number; - - @ForeignKey(() => FolderModel) - @Column - parentId: number; - - @BelongsTo(() => FolderModel) - parent: FolderModel; - - @Index - @Column - name: string; - - @Column(DataType.STRING(24)) - bucket: string; - - @ForeignKey(() => UserModel) - @Column - userId: number; - - @BelongsTo(() => UserModel) - user: UserModel; - - @Column - encryptVersion: '03-aes'; - - @Default(false) - @Column - deleted: boolean; - - @AllowNull - @Column - deletedAt: Date; - - @Column - createdAt: Date; - - @Column - updatedAt: Date; -} +import { FolderModel } from './folder.model'; +import { UserModel } from '../user/user.model'; export interface FolderRepository { findAll(): Promise | []>; diff --git a/src/modules/send/send.module.ts b/src/modules/send/send.module.ts index 1221d88ea..c747a63d4 100644 --- a/src/modules/send/send.module.ts +++ b/src/modules/send/send.module.ts @@ -5,7 +5,7 @@ import { NotificationModule } from '../../externals/notifications/notifications. import { FileModule } from '../file/file.module'; import { FileModel } from '../file/file.repository'; import { FolderModule } from '../folder/folder.module'; -import { FolderModel } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { UserModule } from '../user/user.module'; import { UserModel } from '../user/user.repository'; import { diff --git a/src/modules/send/send.usecase.spec.ts b/src/modules/send/send.usecase.spec.ts index 67a9bf2fc..f570f55c9 100644 --- a/src/modules/send/send.usecase.spec.ts +++ b/src/modules/send/send.usecase.spec.ts @@ -6,7 +6,7 @@ import { Sequelize } from 'sequelize-typescript'; import { CryptoModule } from '../../externals/crypto/crypto.module'; import { NotificationService } from '../../externals/notifications/notification.service'; import { FileModel } from '../file/file.repository'; -import { FolderModel } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { User } from '../user/user.domain'; import { UserModel } from '../user/user.repository'; import { SendLink } from './send-link.domain'; diff --git a/src/modules/share/share.module.ts b/src/modules/share/share.module.ts index 0d9299e89..67c063eb7 100644 --- a/src/modules/share/share.module.ts +++ b/src/modules/share/share.module.ts @@ -4,11 +4,9 @@ import { CryptoModule } from '../../externals/crypto/crypto.module'; import { NotificationModule } from '../../externals/notifications/notifications.module'; import { FileModule } from '../file/file.module'; import { FileModel, SequelizeFileRepository } from '../file/file.repository'; +import { FolderModel } from '../folder/folder.model'; import { FolderModule } from '../folder/folder.module'; -import { - FolderModel, - SequelizeFolderRepository, -} from '../folder/folder.repository'; +import { SequelizeFolderRepository } from '../folder/folder.repository'; import { UserModule } from '../user/user.module'; import { SequelizeUserRepository, UserModel } from '../user/user.repository'; import { ShareController } from './share.controller'; diff --git a/src/modules/share/share.repository.ts b/src/modules/share/share.repository.ts index 99238e807..4c0d8e05e 100644 --- a/src/modules/share/share.repository.ts +++ b/src/modules/share/share.repository.ts @@ -18,7 +18,7 @@ import { import { FileModel } from '../file/file.repository'; import { User, UserAttributes } from '../user/user.domain'; import { UserModel } from '../user/user.repository'; -import { FolderModel } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { Folder, FolderAttributes } from '../folder/folder.domain'; import { Pagination } from '../../lib/pagination'; import { Op } from 'sequelize'; From 32712c0b68b533b14c7291922005f3a197b8c233 Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 15:02:17 +0100 Subject: [PATCH 04/12] moved user model --- src/modules/file/file.repository.ts | 2 +- src/modules/folder/folder.module.ts | 3 +- .../keyserver/key-server.repository.ts | 2 +- src/modules/send/send-link.repository.ts | 2 +- src/modules/send/send.module.ts | 7 +- src/modules/send/send.usecase.spec.ts | 2 +- src/modules/share/share.module.ts | 3 +- src/modules/share/share.repository.ts | 2 +- src/modules/user/user-referrals.repository.ts | 2 +- src/modules/user/user.model.ts | 126 ++++++++++++++++++ src/modules/user/user.module.ts | 2 +- src/modules/user/user.repository.ts | 121 +---------------- .../shared-workspace.repository.ts | 2 +- 13 files changed, 140 insertions(+), 136 deletions(-) create mode 100644 src/modules/user/user.model.ts diff --git a/src/modules/file/file.repository.ts b/src/modules/file/file.repository.ts index 6f12c7810..80602ac23 100644 --- a/src/modules/file/file.repository.ts +++ b/src/modules/file/file.repository.ts @@ -16,7 +16,7 @@ import { PrimaryKey, Table, } from 'sequelize-typescript'; -import { UserModel } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { User } from '../user/user.domain'; import { Folder, FolderAttributes } from '../folder/folder.domain'; import { Pagination } from '../../lib/pagination'; diff --git a/src/modules/folder/folder.module.ts b/src/modules/folder/folder.module.ts index d6ffd95be..43c744c26 100644 --- a/src/modules/folder/folder.module.ts +++ b/src/modules/folder/folder.module.ts @@ -5,7 +5,8 @@ import { FolderModel } from '../folder/folder.model'; import { FolderUseCases } from './folder.usecase'; import { FileModule } from '../file/file.module'; import { CryptoModule } from '../../externals/crypto/crypto.module'; -import { SequelizeUserRepository, UserModel } from '../user/user.repository'; +import { SequelizeUserRepository } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { CryptoService } from '../../externals/crypto/crypto.service'; import { FolderController } from './folder.controller'; diff --git a/src/modules/keyserver/key-server.repository.ts b/src/modules/keyserver/key-server.repository.ts index 3197744bc..235940524 100644 --- a/src/modules/keyserver/key-server.repository.ts +++ b/src/modules/keyserver/key-server.repository.ts @@ -10,7 +10,7 @@ import { Table, } from 'sequelize-typescript'; import { UserAttributes } from '../user/user.domain'; -import { UserModel } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { KeyServer, KeyServerAttributes } from './key-server.domain'; @Table({ diff --git a/src/modules/send/send-link.repository.ts b/src/modules/send/send-link.repository.ts index aba135cda..62e2e7dd7 100644 --- a/src/modules/send/send-link.repository.ts +++ b/src/modules/send/send-link.repository.ts @@ -1,7 +1,7 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { InjectModel } from '@nestjs/sequelize'; import { User } from '../user/user.domain'; -import { UserModel } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { SendLink, SendLinkAttributes } from './send-link.domain'; import { SendLinkItem } from './send-link-item.domain'; diff --git a/src/modules/send/send.module.ts b/src/modules/send/send.module.ts index c747a63d4..06af2aedc 100644 --- a/src/modules/send/send.module.ts +++ b/src/modules/send/send.module.ts @@ -7,12 +7,7 @@ import { FileModel } from '../file/file.repository'; import { FolderModule } from '../folder/folder.module'; import { FolderModel } from '../folder/folder.model'; import { UserModule } from '../user/user.module'; -import { UserModel } from '../user/user.repository'; -import { - SendLinkItemModel, - SendLinkModel, - SequelizeSendRepository, -} from './send-link.repository'; +import { UserModel } from '../user/user.model'; import { SendController } from './send.controller'; import { SendUseCases } from './send.usecase'; diff --git a/src/modules/send/send.usecase.spec.ts b/src/modules/send/send.usecase.spec.ts index f570f55c9..630644a08 100644 --- a/src/modules/send/send.usecase.spec.ts +++ b/src/modules/send/send.usecase.spec.ts @@ -8,7 +8,7 @@ import { NotificationService } from '../../externals/notifications/notification. import { FileModel } from '../file/file.repository'; import { FolderModel } from '../folder/folder.model'; import { User } from '../user/user.domain'; -import { UserModel } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { SendLink } from './send-link.domain'; import { SendLinkItemModel, diff --git a/src/modules/share/share.module.ts b/src/modules/share/share.module.ts index 67c063eb7..ec9f08a3e 100644 --- a/src/modules/share/share.module.ts +++ b/src/modules/share/share.module.ts @@ -7,8 +7,9 @@ import { FileModel, SequelizeFileRepository } from '../file/file.repository'; import { FolderModel } from '../folder/folder.model'; import { FolderModule } from '../folder/folder.module'; import { SequelizeFolderRepository } from '../folder/folder.repository'; +import { UserModel } from '../user/user.model'; import { UserModule } from '../user/user.module'; -import { SequelizeUserRepository, UserModel } from '../user/user.repository'; +import { SequelizeUserRepository } from '../user/user.repository'; import { ShareController } from './share.controller'; import { SequelizeShareRepository, ShareModel } from './share.repository'; import { ShareUseCases } from './share.usecase'; diff --git a/src/modules/share/share.repository.ts b/src/modules/share/share.repository.ts index 4c0d8e05e..758b0afa4 100644 --- a/src/modules/share/share.repository.ts +++ b/src/modules/share/share.repository.ts @@ -17,7 +17,7 @@ import { } from 'sequelize-typescript'; import { FileModel } from '../file/file.repository'; import { User, UserAttributes } from '../user/user.domain'; -import { UserModel } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { FolderModel } from '../folder/folder.model'; import { Folder, FolderAttributes } from '../folder/folder.domain'; import { Pagination } from '../../lib/pagination'; diff --git a/src/modules/user/user-referrals.repository.ts b/src/modules/user/user-referrals.repository.ts index 442e6e3c3..370f08ae5 100644 --- a/src/modules/user/user-referrals.repository.ts +++ b/src/modules/user/user-referrals.repository.ts @@ -13,7 +13,7 @@ import { } from 'sequelize-typescript'; import { ReferralModel } from './referrals.repository'; import { UserReferralAttributes } from './user.domain'; -import { UserModel } from './user.repository'; +import { UserModel } from './user.model'; @Table({ underscored: true, diff --git a/src/modules/user/user.model.ts b/src/modules/user/user.model.ts new file mode 100644 index 000000000..6a1bc1bd8 --- /dev/null +++ b/src/modules/user/user.model.ts @@ -0,0 +1,126 @@ +import { + Table, + Model, + PrimaryKey, + AutoIncrement, + Column, + DataType, + AllowNull, + Unique, + ForeignKey, + BelongsTo, + HasMany, + Default, +} from 'sequelize-typescript'; +import { FolderModel } from '../folder/folder.model'; +import { SendLinkModel } from '../send/send-link.model'; +import { UserAttributes } from './user.domain'; + +@Table({ + underscored: true, + timestamps: true, + tableName: 'users', +}) +export class UserModel extends Model implements UserAttributes { + @PrimaryKey + @AutoIncrement + @Column + id: number; + + @Column(DataType.STRING(60)) + userId: string; + + @Column + name: string; + + @Column + lastname: string; + + @AllowNull(false) + @Column + email: string; + + @Unique + @Column + username: string; + + @Column + bridgeUser: string; + + @Column + password: string; + + @Column + mnemonic: string; + + @ForeignKey(() => FolderModel) + @Column + rootFolderId: number; + + @BelongsTo(() => FolderModel) + rootFolder: FolderModel; + + @HasMany(() => SendLinkModel) + sendLinks: SendLinkModel[]; + + @Column + hKey: Buffer; + + @Column + secret_2FA: string; + + @Column + errorLoginCount: number; + + @Default(false) + @AllowNull + @Column + isEmailActivitySended: number; + + @AllowNull + @Column + referralCode: string; + + @AllowNull + @Column + referrer: string; + + @AllowNull + @Column + syncDate: Date; + + @Unique + @Column + uuid: string; + + @AllowNull + @Column + lastResend: Date; + + @Default(0) + @Column + credit: number; + + @Default(false) + @Column + welcomePack: boolean; + + @Default(true) + @Column + registerCompleted: boolean; + + @AllowNull + @Column + backupsBucket: string; + + @Default(false) + @Column + sharedWorkspace: boolean; + + @Column + tempKey: string; + + @AllowNull + @Column + avatar: string; +} diff --git a/src/modules/user/user.module.ts b/src/modules/user/user.module.ts index d8d35d3bf..01264817b 100644 --- a/src/modules/user/user.module.ts +++ b/src/modules/user/user.module.ts @@ -2,7 +2,7 @@ import { forwardRef, Module } from '@nestjs/common'; import { SequelizeModule } from '@nestjs/sequelize'; import { SequelizeUserRepository } from './user.repository'; import { UserUseCases } from './user.usecase'; -import { UserModel } from './user.repository'; +import { UserModel } from './user.model'; import { FriendInvitationModel, SequelizeSharedWorkspaceRepository, diff --git a/src/modules/user/user.repository.ts b/src/modules/user/user.repository.ts index a32a0cab3..de2a2c5da 100644 --- a/src/modules/user/user.repository.ts +++ b/src/modules/user/user.repository.ts @@ -2,127 +2,8 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/sequelize'; import { User, UserAttributes } from './user.domain'; import { Folder } from '../folder/folder.domain'; -import { - Column, - Model, - Table, - PrimaryKey, - DataType, - Default, - AutoIncrement, - AllowNull, - BelongsTo, - ForeignKey, - Unique, -} from 'sequelize-typescript'; - -import { FolderModel } from '../folder/folder.repository'; import { FindOrCreateOptions, Transaction } from 'sequelize/types'; -@Table({ - underscored: true, - timestamps: true, - tableName: 'users', -}) -export class UserModel extends Model implements UserAttributes { - @PrimaryKey - @AutoIncrement - @Column - id: number; - - @Column(DataType.STRING(60)) - userId: string; - - @Column - name: string; - - @Column - lastname: string; - - @AllowNull(false) - @Column - email: string; - - @Unique - @Column - username: string; - - @Column - bridgeUser: string; - - @Column - password: string; - - @Column - mnemonic: string; - - @ForeignKey(() => FolderModel) - @Column - rootFolderId: number; - - @BelongsTo(() => FolderModel) - rootFolder: FolderModel; - - @Column - hKey: Buffer; - - @Column - secret_2FA: string; - - @Column - errorLoginCount: number; - - @Default(false) - @AllowNull - @Column - isEmailActivitySended: number; - - @AllowNull - @Column - referralCode: string; - - @AllowNull - @Column - referrer: string; - - @AllowNull - @Column - syncDate: Date; - - @Unique - @Column - uuid: string; - - @AllowNull - @Column - lastResend: Date; - - @Default(0) - @Column - credit: number; - - @Default(false) - @Column - welcomePack: boolean; - - @Default(true) - @Column - registerCompleted: boolean; - - @AllowNull - @Column - backupsBucket: string; - - @Default(false) - @Column - sharedWorkspace: boolean; - - @Column - tempKey: string; - - @AllowNull - @Column - avatar: string; -} +import { UserModel } from './user.model'; export interface UserRepository { findById(id: number): Promise; diff --git a/src/shared-workspace/shared-workspace.repository.ts b/src/shared-workspace/shared-workspace.repository.ts index 982ddbaf0..a30f66b89 100644 --- a/src/shared-workspace/shared-workspace.repository.ts +++ b/src/shared-workspace/shared-workspace.repository.ts @@ -12,7 +12,7 @@ import { Table, } from 'sequelize-typescript'; import { UserAttributes } from '../modules/user/user.domain'; -import { UserModel } from '../modules/user/user.repository'; +import { UserModel } from '../modules/user/user.model'; interface FriendInvitationAttributes { id: number; From 50d73df7e8a34591bd2929fe9f0fa93e27444c8c Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 15:09:40 +0100 Subject: [PATCH 05/12] moved send links models --- src/modules/send/send-link-item.model.ts | 57 ++++++++++++ src/modules/send/send-link.model.ts | 65 +++++++++++++ src/modules/send/send-link.repository.ts | 113 +---------------------- src/modules/send/send.module.ts | 3 + 4 files changed, 128 insertions(+), 110 deletions(-) create mode 100644 src/modules/send/send-link-item.model.ts create mode 100644 src/modules/send/send-link.model.ts diff --git a/src/modules/send/send-link-item.model.ts b/src/modules/send/send-link-item.model.ts new file mode 100644 index 000000000..df0d0a580 --- /dev/null +++ b/src/modules/send/send-link-item.model.ts @@ -0,0 +1,57 @@ +import { + Table, + Model, + PrimaryKey, + Column, + ForeignKey, + BelongsTo, + AllowNull, + DataType, +} from 'sequelize-typescript'; +import { SendLinkModel } from './send-link.model'; + +@Table({ + underscored: true, + timestamps: true, + tableName: 'send_links_items', +}) +export class SendLinkItemModel extends Model { + @PrimaryKey + @Column + id: string; + + @Column + name: string; + + @Column + type: string; + + @ForeignKey(() => SendLinkModel) + @Column + linkId: string; + + @BelongsTo(() => SendLinkModel) + link: any; + + @AllowNull + @Column + networkId: string; + + @Column(DataType.STRING(64)) + encryptionKey: string; + + @Column(DataType.INTEGER.UNSIGNED) + size: number; + + @Column + createdAt: Date; + + @Column + updatedAt: Date; + + @Column(DataType.INTEGER) + version: number; + + @Column + parent_folder: string; +} diff --git a/src/modules/send/send-link.model.ts b/src/modules/send/send-link.model.ts new file mode 100644 index 000000000..4277ac765 --- /dev/null +++ b/src/modules/send/send-link.model.ts @@ -0,0 +1,65 @@ +import { + AllowNull, + BelongsTo, + Column, + DataType, + ForeignKey, + HasMany, + Model, + PrimaryKey, + Table, +} from 'sequelize-typescript'; +import { UserModel } from '../user/user.model'; +import { SendLinkItemModel } from './send-link-item.model'; + +@Table({ + underscored: true, + timestamps: true, + tableName: 'send_links', +}) +export class SendLinkModel extends Model { + @PrimaryKey + @Column + id: string; + + @Column + views: number; + + @ForeignKey(() => UserModel) + @Column + userId: number; + + @BelongsTo(() => UserModel) + user: UserModel; + + @Column + sender: string; + + @Column + receivers: string; + + @Column + code: string; + + @Column + title: string; + + @Column + subject: string; + + @Column + expirationAt: Date; + + @Column + createdAt: Date; + + @Column + updatedAt: Date; + + @AllowNull + @Column(DataType.TEXT) + hashedPassword: string; + + @HasMany(() => SendLinkItemModel) + items: SendLinkItemModel[]; +} diff --git a/src/modules/send/send-link.repository.ts b/src/modules/send/send-link.repository.ts index 62e2e7dd7..901969f15 100644 --- a/src/modules/send/send-link.repository.ts +++ b/src/modules/send/send-link.repository.ts @@ -4,122 +4,15 @@ import { User } from '../user/user.domain'; import { UserModel } from '../user/user.model'; import { SendLink, SendLinkAttributes } from './send-link.domain'; import { SendLinkItem } from './send-link-item.domain'; - -import { - Column, - Model, - Table, - PrimaryKey, - ForeignKey, - BelongsTo, - DataType, - AllowNull, - HasMany, - Sequelize, -} from 'sequelize-typescript'; +import { Sequelize } from 'sequelize-typescript'; import { getStringFromBinary, convertStringToBinary, } from '../../lib/binary-converter'; +import { SendLinkModel } from './send-link.model'; +import { SendLinkItemModel } from './send-link-item.model'; const ENCRYPTION_DATE_RELEASE = new Date('2022-07-05 13:55:00'); -@Table({ - underscored: true, - timestamps: true, - tableName: 'send_links', -}) -export class SendLinkModel extends Model { - @PrimaryKey - @Column - id: string; - - @Column - views: number; - - @ForeignKey(() => UserModel) - @Column - userId: number; - - @BelongsTo(() => UserModel) - user: UserModel; - - @Column - sender: string; - - @Column - receivers: string; - - @Column - code: string; - - @Column - title: string; - - @Column - subject: string; - - @Column - expirationAt: Date; - - @Column - createdAt: Date; - - @Column - updatedAt: Date; - - @AllowNull - @Column(DataType.TEXT) - hashedPassword: string; - - @HasMany(() => SendLinkItemModel) - items: SendLinkItemModel[]; -} - -@Table({ - underscored: true, - timestamps: true, - tableName: 'send_links_items', -}) -export class SendLinkItemModel extends Model { - @PrimaryKey - @Column - id: string; - - @Column - name: string; - - @Column - type: string; - - @ForeignKey(() => SendLinkModel) - @Column - linkId: string; - - @BelongsTo(() => SendLinkModel) - link: any; - - @AllowNull - @Column - networkId: string; - - @Column(DataType.STRING(64)) - encryptionKey: string; - - @Column(DataType.INTEGER.UNSIGNED) - size: number; - - @Column - createdAt: Date; - - @Column - updatedAt: Date; - - @Column(DataType.INTEGER) - version: number; - - @Column - parent_folder: string; -} export interface SendRepository { findById(id: SendLinkAttributes['id']): Promise; diff --git a/src/modules/send/send.module.ts b/src/modules/send/send.module.ts index 06af2aedc..d926fc6a9 100644 --- a/src/modules/send/send.module.ts +++ b/src/modules/send/send.module.ts @@ -8,8 +8,11 @@ import { FolderModule } from '../folder/folder.module'; import { FolderModel } from '../folder/folder.model'; import { UserModule } from '../user/user.module'; import { UserModel } from '../user/user.model'; +import { SequelizeSendRepository } from './send-link.repository'; import { SendController } from './send.controller'; import { SendUseCases } from './send.usecase'; +import { SendLinkModel } from './send-link.model'; +import { SendLinkItemModel } from './send-link-item.model'; @Module({ imports: [ From 5d62408c2e19df7117652cedb98a2f457cf9a4c4 Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 15:10:41 +0100 Subject: [PATCH 06/12] fixed model imports on tests --- src/modules/file/file.usecase.spec.ts | 9 ++++----- src/modules/folder/folder.usecase.spec.ts | 5 +++-- src/modules/send/send.usecase.spec.ts | 4 ++-- src/modules/share/share.usecase.spec.ts | 5 +++-- src/modules/trash/trash.usecase.spec.ts | 9 ++++----- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/modules/file/file.usecase.spec.ts b/src/modules/file/file.usecase.spec.ts index 8dd3d2c1e..afbed5808 100644 --- a/src/modules/file/file.usecase.spec.ts +++ b/src/modules/file/file.usecase.spec.ts @@ -16,11 +16,10 @@ import { SequelizeShareRepository, ShareModel, } from '../share/share.repository'; -import { - FolderModel, - SequelizeFolderRepository, -} from '../folder/folder.repository'; -import { SequelizeUserRepository, UserModel } from '../user/user.repository'; +import { SequelizeFolderRepository } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; +import { SequelizeUserRepository } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { BridgeModule } from '../../externals/bridge/bridge.module'; import { BridgeService } from '../../externals/bridge/bridge.service'; import { CryptoService } from '../../externals/crypto/crypto.service'; diff --git a/src/modules/folder/folder.usecase.spec.ts b/src/modules/folder/folder.usecase.spec.ts index ef2fd53f7..24aa65fb8 100644 --- a/src/modules/folder/folder.usecase.spec.ts +++ b/src/modules/folder/folder.usecase.spec.ts @@ -10,7 +10,7 @@ import { } from '@nestjs/common'; import { getModelToken } from '@nestjs/sequelize'; import { Folder, FolderAttributes } from './folder.domain'; -import { FolderModel } from './folder.repository'; +import { FolderModel } from './folder.model'; import { FileUseCases } from '../file/file.usecase'; import { FileModel, SequelizeFileRepository } from '../file/file.repository'; import { @@ -18,7 +18,8 @@ import { ShareModel, } from '../share/share.repository'; import { ShareUseCases } from '../share/share.usecase'; -import { SequelizeUserRepository, UserModel } from '../user/user.repository'; +import { SequelizeUserRepository } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { BridgeModule } from '../../externals/bridge/bridge.module'; import { CryptoModule } from '../../externals/crypto/crypto.module'; import { CryptoService } from '../../externals/crypto/crypto.service'; diff --git a/src/modules/send/send.usecase.spec.ts b/src/modules/send/send.usecase.spec.ts index 630644a08..73bade170 100644 --- a/src/modules/send/send.usecase.spec.ts +++ b/src/modules/send/send.usecase.spec.ts @@ -9,10 +9,10 @@ import { FileModel } from '../file/file.repository'; import { FolderModel } from '../folder/folder.model'; import { User } from '../user/user.domain'; import { UserModel } from '../user/user.model'; +import { SendLinkItemModel } from './send-link-item.model'; import { SendLink } from './send-link.domain'; +import { SendLinkModel } from './send-link.model'; import { - SendLinkItemModel, - SendLinkModel, SendRepository, SequelizeSendRepository, } from './send-link.repository'; diff --git a/src/modules/share/share.usecase.spec.ts b/src/modules/share/share.usecase.spec.ts index e94a8a448..62c8020c3 100644 --- a/src/modules/share/share.usecase.spec.ts +++ b/src/modules/share/share.usecase.spec.ts @@ -17,13 +17,14 @@ import { import { FileUseCases } from '../file/file.usecase'; import { Folder } from '../folder/folder.domain'; import { - FolderModel, FolderRepository, SequelizeFolderRepository, } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { FolderUseCases } from '../folder/folder.usecase'; import { User } from '../user/user.domain'; -import { SequelizeUserRepository, UserModel } from '../user/user.repository'; +import { SequelizeUserRepository } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { UserUseCases } from '../user/user.usecase'; import { Share, ShareAttributes } from './share.domain'; import { SequelizeShareRepository, ShareModel } from './share.repository'; diff --git a/src/modules/trash/trash.usecase.spec.ts b/src/modules/trash/trash.usecase.spec.ts index 9454394fe..c67fefaf6 100644 --- a/src/modules/trash/trash.usecase.spec.ts +++ b/src/modules/trash/trash.usecase.spec.ts @@ -2,13 +2,12 @@ import { Test, TestingModule } from '@nestjs/testing'; import { TrashUseCases } from './trash.usecase'; import { FileModel, SequelizeFileRepository } from '../file/file.repository'; import { File, FileAttributes } from '../file/file.domain'; -import { - FolderModel, - SequelizeFolderRepository, -} from '../folder/folder.repository'; +import { SequelizeFolderRepository } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { getModelToken } from '@nestjs/sequelize'; import { User } from '../user/user.domain'; -import { SequelizeUserRepository, UserModel } from '../user/user.repository'; +import { SequelizeUserRepository } from '../user/user.repository'; +import { UserModel } from '../user/user.model'; import { Folder, FolderAttributes } from '../folder/folder.domain'; import { FileUseCases } from '../file/file.usecase'; import { FolderUseCases } from '../folder/folder.usecase'; From d0fda468716c53e4ec2e2879dcd9b2e17bb212a1 Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 15:12:10 +0100 Subject: [PATCH 07/12] fixed winston logger bug on dev run --- src/config/initializers/database.ts | 2 +- src/lib/winston-logger.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/config/initializers/database.ts b/src/config/initializers/database.ts index b87b82efe..64cb7cae1 100644 --- a/src/config/initializers/database.ts +++ b/src/config/initializers/database.ts @@ -50,7 +50,7 @@ export default class Database { instance .authenticate() .then(() => { - logger.info('Connected to database'); + logger.log('Connected to database'); }) .catch((err) => { logger.error('Database connection error: %s', err); diff --git a/src/lib/winston-logger.ts b/src/lib/winston-logger.ts index aad96882b..21a6a3777 100644 --- a/src/lib/winston-logger.ts +++ b/src/lib/winston-logger.ts @@ -1,6 +1,7 @@ import { WinstonModule } from 'nest-winston'; import winston from 'winston'; import os from 'os'; +import { LoggerService } from '@nestjs/common'; const { splat, combine, printf, timestamp } = winston.format; const serverHostname = os.hostname(); @@ -46,12 +47,12 @@ export class WinstonLogger { transports: [new winston.transports.Console()], }; - static getLogger() { + static getLogger(): LoggerService { const configs = { production: WinstonModule.createLogger(this.prodOptions), staging: WinstonModule.createLogger(this.prodOptions), - development: null, + development: WinstonModule.createLogger(this.devOptions), }; - return configs[process.env.NODE_EV] || null; + return configs[process.env.NODE_EV] || configs.development; } } From 5ce07be5f70d119030421ec0eb6eb79747a01207 Mon Sep 17 00:00:00 2001 From: larry-internxt Date: Mon, 13 Feb 2023 18:10:03 +0100 Subject: [PATCH 08/12] added script to move expired send links to deleted files --- bin/expired-send-links/index.ts | 185 ++++++++++++++++++++++++++++++++ bin/expired-send-links/utils.ts | 86 +++++++++++++++ package.json | 1 + yarn.lock | 5 + 4 files changed, 277 insertions(+) create mode 100644 bin/expired-send-links/index.ts create mode 100644 bin/expired-send-links/utils.ts diff --git a/bin/expired-send-links/index.ts b/bin/expired-send-links/index.ts new file mode 100644 index 000000000..08862341d --- /dev/null +++ b/bin/expired-send-links/index.ts @@ -0,0 +1,185 @@ +import { Command } from 'commander'; +import Database from '../../src/config/initializers/database'; +import { UserModel } from '../../src/modules/user/user.model'; +import { FolderModel } from '../../src/modules/folder/folder.model'; +import { SendLinkItemModel } from '../../src/modules/send/send-link-item.model'; +import { DeletedFileModel } from '../../src/modules/deleted-file/deleted-file.model'; +import { SendLinkModel } from '../../src/modules/send/send-link.model'; +import { SendLinkAttributes } from '../../src/modules/send/send-link.domain'; +import { ModelType } from 'sequelize-typescript'; +import { + createTimer, + getExpiredSendLinks, + moveToDeletedFiles, + clearExpiredSendLink, + clearExpiredSendLinkItems, +} from './utils'; + +const commands: { flags: string; description: string; required: boolean }[] = [ + { + flags: '--db-hostname ', + description: 'The hostname of the database where deleted files are stored', + required: true, + }, + { + flags: '--db-name ', + description: 'The name of the database where deleted files are stored', + required: true, + }, + { + flags: '--db-username ', + description: + 'The username authorized to read and delete from the deleted files table', + required: true, + }, + { + flags: '--db-password ', + description: 'The database username password', + required: true, + }, + { + flags: '--db-port ', + description: 'The database port', + required: true, + }, + { + flags: '--send-userid ', + description: 'The user id from Send', + required: true, + }, + { + flags: '--send-folderid ', + description: 'The folder id from Send user where files are stored', + required: true, + }, + { + flags: '--send-bucketid ', + description: 'The bucket id from Send user', + required: true, + }, + { + flags: '-l, --limit ', + description: 'The files limit to handle each time', + required: false, + }, +]; + +const command = new Command('expired-send-links').version('0.0.1'); + +commands.forEach((c) => { + if (c.required) { + command.requiredOption(c.flags, c.description); + } else { + command.option(c.flags, c.description); + } +}); + +command.parse(process.argv); + +const opts = command.opts(); +const db = Database.getInstance({ + sequelizeConfig: { + host: opts.dbHostname, + port: opts.dbPort, + database: opts.dbName, + username: opts.dbUsername, + password: opts.dbPassword, + dialect: 'postgres', + repositoryMode: true, + dialectOptions: { + ssl: { + require: true, + rejectUnauthorized: false, + }, + }, + }, +}); +db.addModels([ + SendLinkModel, + UserModel, + SendLinkItemModel, + FolderModel, + DeletedFileModel, +]); + +const timer = createTimer(); +timer.start(); + +let totalMovedExpiredLinks = 0; + +const logIntervalId = setInterval(() => { + console.log( + 'EXPIRED LINKS DELETED RATE: %s/s', + totalMovedExpiredLinks / (timer.end() / 1000), + ); +}, 10000); + +function finishProgram() { + clearInterval(logIntervalId); + + console.log( + 'TOTAL EXPIRED LINKS DELETED %s | DURATION %ss', + totalMovedExpiredLinks, + (timer.end() / 1000).toFixed(2), + ); + db.close() + .then(() => { + console.log('DISCONNECTED FROM DB'); + }) + .catch((err) => { + console.log( + 'Error closing connection %s. %s', + err.message.err.stack || 'NO STACK.', + ); + }); +} + +process.on('SIGINT', () => finishProgram()); + +async function start(limit = 20) { + const SendLinkRepository = db.getRepository(SendLinkModel); + const SendLinkItemRepository = db.getRepository(SendLinkItemModel); + const DeletedFilesRepository = db.getRepository(DeletedFileModel); + const sendLinkItemModel = db.models.SendLinkItemModel as unknown as ModelType< + SendLinkItemModel, + SendLinkItemModel + >; + + let expiredLinks: SendLinkAttributes[] = []; + + do { + expiredLinks = await getExpiredSendLinks( + SendLinkRepository, + sendLinkItemModel, + limit, + ); + + console.time('df-network-req'); + + for (const expiredLink of expiredLinks) { + for (const expiredLinkItem of expiredLink.items) { + await moveToDeletedFiles( + DeletedFilesRepository, + expiredLinkItem, + Number(opts.sendUserid), + Number(opts.sendFolderid), + String(opts.sendBucketid), + ); + } + await clearExpiredSendLinkItems(SendLinkItemRepository, expiredLink); + await clearExpiredSendLink(SendLinkRepository, expiredLink); + } + + console.timeEnd('df-network-req'); + + totalMovedExpiredLinks += expiredLinks.length; + } while (expiredLinks.length === limit); +} + +start(parseInt(opts.limit || '20')) + .catch((err) => { + console.log('err', err); + }) + .finally(() => { + finishProgram(); + }); diff --git a/bin/expired-send-links/utils.ts b/bin/expired-send-links/utils.ts new file mode 100644 index 000000000..c4fcbea43 --- /dev/null +++ b/bin/expired-send-links/utils.ts @@ -0,0 +1,86 @@ +import { Op } from 'sequelize'; +import { ModelType, Repository, Sequelize } from 'sequelize-typescript'; +import { DeletedFileModel } from 'src/modules/deleted-file/deleted-file.model'; +import { SendLinkItemAttributes } from '../../src/modules/send/send-link-item.domain'; +import { SendLinkItemModel } from '../../src/modules/send/send-link-item.model'; +import { SendLinkAttributes } from '../../src/modules/send/send-link.domain'; +import { SendLinkModel } from '../../src/modules/send/send-link.model'; + +type Timer = { start: () => void; end: () => number }; + +export const createTimer = (): Timer => { + let timeStart: [number, number]; + + return { + start: () => { + timeStart = process.hrtime(); + }, + end: () => { + const NS_PER_SEC = 1e9; + const NS_TO_MS = 1e6; + const diff = process.hrtime(timeStart); + + return (diff[0] * NS_PER_SEC + diff[1]) / NS_TO_MS; + }, + }; +}; + +export function getExpiredSendLinks( + SendLinkRepository: Repository, + SendLinkItemModel: ModelType, + limit: number, +): Promise { + return SendLinkRepository.findAll({ + limit, + order: [['id', 'ASC']], + where: { + expirationAt: { + [Op.lt]: Sequelize.literal('NOW()'), + }, + }, + include: { + model: SendLinkItemModel, + attributes: ['id', 'networkId'], + where: { + networkId: { + [Op.not]: null, + }, + }, + }, + }).then((res) => { + return res as unknown as SendLinkAttributes[]; + }); +} + +export function moveToDeletedFiles( + DeletedFilesRepository: Repository, + expiredSendLink: SendLinkItemAttributes, + SEND_USER_ID: number, + SEND_FOLDER_ID: number, + SEND_BUCKET: string, +): Promise { + return DeletedFilesRepository.create({ + file_id: expiredSendLink.networkId, + user_id: SEND_USER_ID, + folder_id: SEND_FOLDER_ID, + bucket: SEND_BUCKET, + }); +} + +export function clearExpiredSendLinkItems( + SendLinkItemRepository: Repository, + expiredLink: SendLinkAttributes, +) { + return SendLinkItemRepository.destroy({ + where: { link_id: expiredLink.id }, + }); +} + +export function clearExpiredSendLink( + SendLinkRepository: Repository, + expiredLink: SendLinkAttributes, +) { + return SendLinkRepository.destroy({ + where: { id: expiredLink.id }, + }); +} diff --git a/package.json b/package.json index 9d10de9fc..42b40bdd8 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "@types/uuid": "^8.3.4", "@typescript-eslint/eslint-plugin": "^5.0.0", "@typescript-eslint/parser": "^5.0.0", + "commander": "^10.0.0", "eslint": "^8.0.1", "eslint-config-prettier": "^8.3.0", "eslint-plugin-prettier": "^4.0.0", diff --git a/yarn.lock b/yarn.lock index ac95f1cc3..f9b9efbef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2522,6 +2522,11 @@ commander@4.1.1: resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068" integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA== +commander@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1" + integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA== + commander@^2.19.0, commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" From 871d191443b92d38e6231ac82939aff94517518b Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Tue, 14 Feb 2023 16:53:28 +0100 Subject: [PATCH 09/12] refactor(bin/remove-send-expired-links): * Use the new optional/required convention for commander. --- bin/expired-send-links/index.ts | 37 +++++++++++---------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/bin/expired-send-links/index.ts b/bin/expired-send-links/index.ts index 08862341d..b51adbc77 100644 --- a/bin/expired-send-links/index.ts +++ b/bin/expired-send-links/index.ts @@ -15,63 +15,50 @@ import { clearExpiredSendLinkItems, } from './utils'; -const commands: { flags: string; description: string; required: boolean }[] = [ +const commands: { flags: string; description: string }[] = [ { flags: '--db-hostname ', - description: 'The hostname of the database where deleted files are stored', - required: true, + description: 'The database hostname', }, { flags: '--db-name ', - description: 'The name of the database where deleted files are stored', - required: true, + description: 'The database name', }, { flags: '--db-username ', description: - 'The username authorized to read and delete from the deleted files table', - required: true, + 'The username authorized to create from deleted_files table and read/delete from send_links and send_link_items', }, { flags: '--db-password ', description: 'The database username password', - required: true, }, { flags: '--db-port ', description: 'The database port', - required: true, }, { - flags: '--send-userid ', - description: 'The user id from Send', - required: true, + flags: '--user-id ', + description: 'The user owner of the files', }, { - flags: '--send-folderid ', - description: 'The folder id from Send user where files are stored', - required: true, + flags: '--folder-id ', + description: 'The folder id where files are stored', }, { - flags: '--send-bucketid ', - description: 'The bucket id from Send user', - required: true, + flags: '--bucket-id ', + description: 'The bucket id where the files are stored in the network', }, { - flags: '-l, --limit ', + flags: '-l, --limit [limit]', description: 'The files limit to handle each time', - required: false, }, ]; const command = new Command('expired-send-links').version('0.0.1'); commands.forEach((c) => { - if (c.required) { - command.requiredOption(c.flags, c.description); - } else { - command.option(c.flags, c.description); - } + command.option(c.flags, c.description); }); command.parse(process.argv); From fe906a0df0b33317a71678e74f5aa90df76fc8e0 Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Tue, 14 Feb 2023 16:54:46 +0100 Subject: [PATCH 10/12] feat(bin/remove-send-expired-links): * Move items in bulk to deleted_files table --- bin/expired-send-links/index.ts | 12 ++++++------ bin/expired-send-links/utils.ts | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/bin/expired-send-links/index.ts b/bin/expired-send-links/index.ts index b51adbc77..37a9d6bf9 100644 --- a/bin/expired-send-links/index.ts +++ b/bin/expired-send-links/index.ts @@ -10,7 +10,7 @@ import { ModelType } from 'sequelize-typescript'; import { createTimer, getExpiredSendLinks, - moveToDeletedFiles, + moveItemsToDeletedFiles, clearExpiredSendLink, clearExpiredSendLinkItems, } from './utils'; @@ -141,13 +141,13 @@ async function start(limit = 20) { limit, ); - console.time('df-network-req'); + console.time('move-to-deleted-files'); for (const expiredLink of expiredLinks) { - for (const expiredLinkItem of expiredLink.items) { - await moveToDeletedFiles( + for (let i = 0; i < expiredLink.items.length; i += 20) { + await moveItemsToDeletedFiles( DeletedFilesRepository, - expiredLinkItem, + expiredLink.items.slice(i, i + 20), Number(opts.sendUserid), Number(opts.sendFolderid), String(opts.sendBucketid), @@ -157,7 +157,7 @@ async function start(limit = 20) { await clearExpiredSendLink(SendLinkRepository, expiredLink); } - console.timeEnd('df-network-req'); + console.timeEnd('move-to-deleted-files'); totalMovedExpiredLinks += expiredLinks.length; } while (expiredLinks.length === limit); diff --git a/bin/expired-send-links/utils.ts b/bin/expired-send-links/utils.ts index c4fcbea43..bdb72c047 100644 --- a/bin/expired-send-links/utils.ts +++ b/bin/expired-send-links/utils.ts @@ -52,6 +52,31 @@ export function getExpiredSendLinks( }); } +export function moveItemsToDeletedFiles( + deletedFilesRepository: Repository, + expiredSendLinkItems: SendLinkItemAttributes[], + userId: number, + folderId: number, + bucketId: string, +): Promise { + const deletedFiles: { + file_id: string; + user_id: number; + folder_id: number; + bucket: string; + }[] = expiredSendLinkItems.map((item) => { + return { + file_id: item.networkId, + user_id: userId, + folder_id: folderId, + bucket: bucketId, + }; + }); + + + return deletedFilesRepository.bulkCreate(deletedFiles); +} + export function moveToDeletedFiles( DeletedFilesRepository: Repository, expiredSendLink: SendLinkItemAttributes, From 997bb47bb2f18a9bf728ac257c24be04a65521bd Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Wed, 15 Feb 2023 09:27:31 +0100 Subject: [PATCH 11/12] feat(send): Dockerfile for clearing expired send links command --- bin/expired-send-links/Dockerfile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 bin/expired-send-links/Dockerfile diff --git a/bin/expired-send-links/Dockerfile b/bin/expired-send-links/Dockerfile new file mode 100644 index 000000000..3fe844d8c --- /dev/null +++ b/bin/expired-send-links/Dockerfile @@ -0,0 +1,15 @@ +FROM mhart/alpine-node:16 +LABEL author="internxt" + +WORKDIR / + +# Add useful packages +RUN apk add git curl + +COPY . . + +# Install deps +RUN yarn + +# Run command +CMD ts-node ./bin/expired-send-links/index.ts From 9d5d591aa925ec613582137192e8719a0088bb55 Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Wed, 15 Feb 2023 09:28:38 +0100 Subject: [PATCH 12/12] fix(bin/remove-send-expired-links): * Refer properly options with short format --- bin/expired-send-links/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/expired-send-links/index.ts b/bin/expired-send-links/index.ts index 37a9d6bf9..e6ab35902 100644 --- a/bin/expired-send-links/index.ts +++ b/bin/expired-send-links/index.ts @@ -148,9 +148,9 @@ async function start(limit = 20) { await moveItemsToDeletedFiles( DeletedFilesRepository, expiredLink.items.slice(i, i + 20), - Number(opts.sendUserid), - Number(opts.sendFolderid), - String(opts.sendBucketid), + Number(opts.userId), + Number(opts.folderId), + String(opts.bucketId), ); } await clearExpiredSendLinkItems(SendLinkItemRepository, expiredLink);