From 6ac6a8d6d63b331804bb63b17056a5f1a24e3352 Mon Sep 17 00:00:00 2001 From: pengx17 Date: Thu, 23 Jan 2025 07:12:16 +0000 Subject: [PATCH] feat(core): unused blob management in settings (#9795) fix AF-2144, PD-2064, PD-2065, PD-2066 --- .../block-attachment/src/attachment-block.ts | 4 +- .../affine/components/src/icons/file-icons.ts | 2 +- .../nodes/footnote-node/footnote-popup.ts | 4 +- .../components/attachment-viewer-panel.ts | 4 +- .../electron/src/helper/workspace/handlers.ts | 15 +- .../frontend/component/src/hooks/index.ts | 1 + .../component/src/hooks/use-disposable.ts | 60 ++++ .../frontend/component/src/lit-react/index.ts | 1 + .../frontend/component/src/lit-react/utils.ts | 9 + .../ai/chat-panel/components/file-chip.ts | 4 +- .../setting/setting-sidebar/style.css.ts | 2 + .../setting/workspace-setting/index.tsx | 2 +- .../storage/blob-management.tsx | 276 ++++++++++++++++++ .../workspace-setting/storage/index.tsx | 24 +- .../workspace-setting/storage/style.css.ts | 133 +++++++++ .../storage/workspace-quota.tsx | 2 +- .../blob-management/entity/unused-blobs.ts | 164 +++++++++++ .../core/src/modules/blob-management/index.ts | 18 ++ .../modules/blob-management/services/index.ts | 11 + packages/frontend/core/src/modules/index.ts | 2 + .../modules/workspace-engine/impls/cloud.ts | 26 +- .../modules/workspace-engine/impls/local.ts | 28 ++ .../modules/workspace/providers/flavour.ts | 14 +- packages/frontend/i18n/src/resources/en.json | 7 +- tests/affine-cloud/e2e/storage.spec.ts | 61 ++++ tests/affine-desktop/e2e/workspace.spec.ts | 2 + 26 files changed, 846 insertions(+), 30 deletions(-) create mode 100644 packages/frontend/component/src/hooks/use-disposable.ts create mode 100644 packages/frontend/component/src/lit-react/utils.ts create mode 100644 packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/blob-management.tsx create mode 100644 packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts create mode 100644 packages/frontend/core/src/modules/blob-management/index.ts create mode 100644 packages/frontend/core/src/modules/blob-management/services/index.ts create mode 100644 tests/affine-cloud/e2e/storage.spec.ts diff --git a/blocksuite/affine/block-attachment/src/attachment-block.ts b/blocksuite/affine/block-attachment/src/attachment-block.ts index 5e75041acc597..16c8eb97735e3 100644 --- a/blocksuite/affine/block-attachment/src/attachment-block.ts +++ b/blocksuite/affine/block-attachment/src/attachment-block.ts @@ -3,7 +3,7 @@ import { CaptionedBlockComponent } from '@blocksuite/affine-components/caption'; import { HoverController } from '@blocksuite/affine-components/hover'; import { AttachmentIcon16, - getAttachmentFileIcons, + getAttachmentFileIcon, } from '@blocksuite/affine-components/icons'; import { Peekable } from '@blocksuite/affine-components/peek'; import { toast } from '@blocksuite/affine-components/toast'; @@ -226,7 +226,7 @@ export class AttachmentBlockComponent extends CaptionedBlockComponent< const infoText = this.error ? 'File loading failed.' : humanFileSize(size); const fileType = name.split('.').pop() ?? ''; - const FileTypeIcon = getAttachmentFileIcons(fileType); + const FileTypeIcon = getAttachmentFileIcon(fileType); const embedView = this.embedView; diff --git a/blocksuite/affine/components/src/icons/file-icons.ts b/blocksuite/affine/components/src/icons/file-icons.ts index c9d76b6243bb7..a96492949eaad 100644 --- a/blocksuite/affine/components/src/icons/file-icons.ts +++ b/blocksuite/affine/components/src/icons/file-icons.ts @@ -1,6 +1,6 @@ import { html } from 'lit'; -export function getAttachmentFileIcons(filetype: string) { +export function getAttachmentFileIcon(filetype: string) { switch (filetype) { case 'img': return IMGFileIcon; diff --git a/blocksuite/affine/components/src/rich-text/inline/presets/nodes/footnote-node/footnote-popup.ts b/blocksuite/affine/components/src/rich-text/inline/presets/nodes/footnote-node/footnote-popup.ts index 617213a92ea37..f0f9483293a7c 100644 --- a/blocksuite/affine/components/src/rich-text/inline/presets/nodes/footnote-node/footnote-popup.ts +++ b/blocksuite/affine/components/src/rich-text/inline/presets/nodes/footnote-node/footnote-popup.ts @@ -7,7 +7,7 @@ import { DualLinkIcon, LinkIcon } from '@blocksuite/icons/lit'; import { css, html, LitElement, type TemplateResult } from 'lit'; import { property } from 'lit/decorators.js'; -import { getAttachmentFileIcons } from '../../../../../icons'; +import { getAttachmentFileIcon } from '../../../../../icons'; import { RefNodeSlotsProvider } from '../../../../extension/ref-node-slots'; export class FootNotePopup extends WithDisposable(LitElement) { @@ -34,7 +34,7 @@ export class FootNotePopup extends WithDisposable(LitElement) { if (!fileType) { return undefined; } - return getAttachmentFileIcons(fileType); + return getAttachmentFileIcon(fileType); } return undefined; }; diff --git a/blocksuite/playground/apps/_common/components/attachment-viewer-panel.ts b/blocksuite/playground/apps/_common/components/attachment-viewer-panel.ts index 644d1a50a2d1a..baac5266f3149 100644 --- a/blocksuite/playground/apps/_common/components/attachment-viewer-panel.ts +++ b/blocksuite/playground/apps/_common/components/attachment-viewer-panel.ts @@ -1,7 +1,7 @@ /* oxlint-disable @typescript-eslint/no-non-null-assertion */ import type { AttachmentBlockModel } from '@blocksuite/affine-model'; import { humanFileSize } from '@blocksuite/affine-shared/utils'; -import { getAttachmentFileIcons } from '@blocksuite/blocks'; +import { getAttachmentFileIcon } from '@blocksuite/blocks'; import { SignalWatcher, WithDisposable } from '@blocksuite/global/utils'; import { ArrowDownBigIcon, @@ -155,7 +155,7 @@ export class AttachmentViewerPanel extends SignalWatcher( const { name, size } = model; const fileType = name.split('.').pop() ?? ''; - const icon = getAttachmentFileIcons(fileType); + const icon = getAttachmentFileIcon(fileType); const isPDF = fileType === 'pdf'; this.#fileInfo.value = { diff --git a/packages/frontend/apps/electron/src/helper/workspace/handlers.ts b/packages/frontend/apps/electron/src/helper/workspace/handlers.ts index cf5b0872c69a7..111f2a63a8f94 100644 --- a/packages/frontend/apps/electron/src/helper/workspace/handlers.ts +++ b/packages/frontend/apps/electron/src/helper/workspace/handlers.ts @@ -8,7 +8,6 @@ import { import fs from 'fs-extra'; import { applyUpdate, Doc as YDoc } from 'yjs'; -import { isWindows } from '../../shared/utils'; import { logger } from '../logger'; import { getDocStoragePool } from '../nbstore'; import { ensureSQLiteDisconnected } from '../nbstore/v1/ensure-db'; @@ -69,16 +68,10 @@ export async function trashWorkspace(universalId: string) { await fs.ensureDir(movedPath); // todo(@pengx17): it seems the db file is still being used at the point // on windows so that it cannot be moved. we will fallback to copy the dir instead. - if (isWindows()) { - await fs.copy(path.dirname(dbPath), movedPath, { - overwrite: true, - }); - await fs.rmdir(path.dirname(dbPath), { recursive: true }); - } else { - return await fs.move(path.dirname(dbPath), movedPath, { - overwrite: true, - }); - } + await fs.copy(path.dirname(dbPath), movedPath, { + overwrite: true, + }); + await fs.rmdir(path.dirname(dbPath), { recursive: true }); } catch (error) { logger.error('trashWorkspace', error); } diff --git a/packages/frontend/component/src/hooks/index.ts b/packages/frontend/component/src/hooks/index.ts index 1655abe94f3e8..7720615b08c03 100644 --- a/packages/frontend/component/src/hooks/index.ts +++ b/packages/frontend/component/src/hooks/index.ts @@ -1,4 +1,5 @@ export { useAutoFocus, useAutoSelect } from './focus-and-select'; +export { useDisposable } from './use-disposable'; export { useRefEffect } from './use-ref-effect'; export * from './use-theme-color-meta'; export * from './use-theme-value'; diff --git a/packages/frontend/component/src/hooks/use-disposable.ts b/packages/frontend/component/src/hooks/use-disposable.ts new file mode 100644 index 0000000000000..491c965f8cc4b --- /dev/null +++ b/packages/frontend/component/src/hooks/use-disposable.ts @@ -0,0 +1,60 @@ +import { useEffect, useState } from 'react'; + +export function useDisposable( + disposableFn: (abortSignal?: AbortSignal) => Promise, + deps?: any[] +): { data: T | null; loading: boolean; error: Error | null }; + +export function useDisposable( + disposableFn: (abortSignal?: AbortSignal) => T | null, + deps?: any[] +): { data: T | null }; + +export function useDisposable( + disposableFn: (abortSignal?: AbortSignal) => Promise | T | null, + deps?: any[] +) { + const [state, setState] = useState<{ + data: T | null; + loading: boolean; + error: Error | null; + }>({ + data: null, + loading: false, + error: null, + }); + + useEffect(() => { + const abortController = new AbortController(); + let _data: T | null = null; + setState(prev => ({ ...prev, loading: true, error: null })); + + Promise.resolve(disposableFn(abortController.signal)) + .then(data => { + _data = data; + if (!abortController.signal.aborted) { + setState({ data, loading: false, error: null }); + } + }) + .catch(error => { + if (!abortController.signal.aborted) { + setState(prev => ({ ...prev, error, loading: false })); + } + }); + + return () => { + abortController.abort(); + + if (_data && typeof _data === 'object') { + if (Symbol.dispose in _data) { + _data[Symbol.dispose](); + } else if (Symbol.asyncDispose in _data) { + _data[Symbol.asyncDispose](); + } + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps || []); + + return state; +} diff --git a/packages/frontend/component/src/lit-react/index.ts b/packages/frontend/component/src/lit-react/index.ts index fa5dc3777b9d8..fb388ef511384 100644 --- a/packages/frontend/component/src/lit-react/index.ts +++ b/packages/frontend/component/src/lit-react/index.ts @@ -1,3 +1,4 @@ export { createComponent as createReactComponentFromLit } from './create-component'; export * from './lit-portal'; export { toReactNode } from './to-react-node'; +export { templateToString } from './utils'; diff --git a/packages/frontend/component/src/lit-react/utils.ts b/packages/frontend/component/src/lit-react/utils.ts new file mode 100644 index 0000000000000..643ca852dd367 --- /dev/null +++ b/packages/frontend/component/src/lit-react/utils.ts @@ -0,0 +1,9 @@ +import type { TemplateResult } from 'lit'; + +export function templateToString({ strings, values }: TemplateResult): string { + return strings.reduce( + (result, str, i) => + result + str + (i < values.length ? String(values[i]) : ''), + '' + ); +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts index 7ba56f1e03cb8..a8b0e90f64b1d 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts @@ -1,5 +1,5 @@ import { ShadowlessElement } from '@blocksuite/affine/block-std'; -import { getAttachmentFileIcons } from '@blocksuite/affine/blocks'; +import { getAttachmentFileIcon } from '@blocksuite/affine/blocks'; import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils'; import { html } from 'lit'; import { property } from 'lit/decorators.js'; @@ -17,7 +17,7 @@ export class ChatPanelFileChip extends SignalWatcher( const { state, fileName, fileType } = this.chip; const isLoading = state === 'embedding' || state === 'uploading'; const tooltip = getChipTooltip(state, fileName, this.chip.tooltip); - const fileIcon = getAttachmentFileIcons(fileType); + const fileIcon = getAttachmentFileIcon(fileType); const icon = getChipIcon(state, fileIcon); return html`; case 'workspace:storage': - return ; + return ; default: return null; } diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/blob-management.tsx b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/blob-management.tsx new file mode 100644 index 0000000000000..2f4fdf5a53799 --- /dev/null +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/blob-management.tsx @@ -0,0 +1,276 @@ +import { + Button, + Checkbox, + Loading, + templateToString, + useConfirmModal, + useDisposable, +} from '@affine/component'; +import { Pagination } from '@affine/component/member-components'; +import { BlobManagementService } from '@affine/core/modules/blob-management/services'; +import { useI18n } from '@affine/i18n'; +import type { ListedBlobRecord } from '@affine/nbstore'; +import { getAttachmentFileIcon } from '@blocksuite/affine/blocks'; +import { DeleteIcon } from '@blocksuite/icons/rc'; +import { useLiveData, useService } from '@toeverything/infra'; +import bytes from 'bytes'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +import * as styles from './style.css'; + +const Empty = () => { + const t = useI18n(); + return ( +
+ {t['com.affine.settings.workspace.storage.unused-blobs.empty']()} +
+ ); +}; + +const useBlob = (blobRecord: ListedBlobRecord) => { + const unusedBlobsEntity = useService(BlobManagementService).unusedBlobs; + return useDisposable( + (abortSignal?: AbortSignal) => + unusedBlobsEntity.hydrateBlob(blobRecord, abortSignal), + [blobRecord] + ); +}; + +const BlobPreview = ({ blobRecord }: { blobRecord: ListedBlobRecord }) => { + const { data, loading, error } = useBlob(blobRecord); + + const element = useMemo(() => { + if (loading) return ; + if (!data?.url || !data.type) return null; + + const { url, type, mime } = data; + + const icon = templateToString(getAttachmentFileIcon(type)); + + if (error) { + return ( +
+ ); + } + + if (mime?.startsWith('image/')) { + return ( + {blobRecord.key} + ); + } else { + return ( +
+ ); + } + }, [loading, data, error, blobRecord.key]); + + return ( +
+
{element}
+
+
{blobRecord.key}
+
+ {data?.type} ยท {bytes(blobRecord.size)} +
+
+
+ ); +}; + +const BlobCard = ({ + blobRecord, + onClick, + selected, +}: { + blobRecord: ListedBlobRecord; + onClick: () => void; + selected: boolean; +}) => { + return ( +
+ + +
+ ); +}; + +const PAGE_SIZE = 9; + +export const BlobManagementPanel = () => { + const t = useI18n(); + + const unusedBlobsEntity = useService(BlobManagementService).unusedBlobs; + const originalUnusedBlobs = useLiveData(unusedBlobsEntity.unusedBlobs$); + const isLoading = useLiveData(unusedBlobsEntity.isLoading$); + const [pageNum, setPageNum] = useState(0); + const [skip, setSkip] = useState(0); + + const [unusedBlobs, setUnusedBlobs] = useState([]); + const unusedBlobsPage = useMemo(() => { + return unusedBlobs.slice(skip, skip + PAGE_SIZE); + }, [unusedBlobs, skip]); + + useEffect(() => { + setUnusedBlobs(originalUnusedBlobs); + }, [originalUnusedBlobs]); + + useEffect(() => { + unusedBlobsEntity.revalidate(); + }, [unusedBlobsEntity]); + + const [selectedBlobs, setSelectedBlobs] = useState([]); + const [deleting, setDeleting] = useState(false); + + const handleSelectBlob = useCallback((blob: ListedBlobRecord) => { + setSelectedBlobs(prev => { + if (prev.includes(blob)) { + return prev; + } + return [...prev, blob]; + }); + }, []); + + const handleUnselectBlob = useCallback((blob: ListedBlobRecord) => { + setSelectedBlobs(prev => prev.filter(b => b.key !== blob.key)); + }, []); + + const handleSelectAll = useCallback(() => { + unusedBlobsPage.forEach(blob => handleSelectBlob(blob)); + }, [unusedBlobsPage, handleSelectBlob]); + + const { openConfirmModal } = useConfirmModal(); + + const handleDeleteSelectedBlobs = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + const currentSelectedBlobs = selectedBlobs; + openConfirmModal({ + title: + t[ + 'com.affine.settings.workspace.storage.unused-blobs.delete.title' + ](), + children: + t[ + 'com.affine.settings.workspace.storage.unused-blobs.delete.warning' + ](), + onConfirm: async () => { + setDeleting(true); + for (const blob of currentSelectedBlobs) { + await unusedBlobsEntity.deleteBlob(blob.key, true); + handleUnselectBlob(blob); + setUnusedBlobs(prev => prev.filter(b => b.key !== blob.key)); + } + setDeleting(false); + }, + confirmText: t['Delete'](), + cancelText: t['Cancel'](), + confirmButtonOptions: { + variant: 'error', + }, + }); + }, + [selectedBlobs, openConfirmModal, t, unusedBlobsEntity, handleUnselectBlob] + ); + + const blobPreviewGridRef = useRef(null); + + useEffect(() => { + if (blobPreviewGridRef.current) { + const unselectBlobs = (e: MouseEvent) => { + const target = e.target as HTMLElement; + if ( + !blobPreviewGridRef.current?.contains(target) && + !target.closest('modal-transition-container') + ) { + setSelectedBlobs([]); + } + }; + document.addEventListener('click', unselectBlobs); + return () => { + document.removeEventListener('click', unselectBlobs); + }; + } + return; + }, [unusedBlobs]); + + return ( + <> + {selectedBlobs.length > 0 ? ( +
+
+ {`${selectedBlobs.length} ${t['com.affine.settings.workspace.storage.unused-blobs.selected']()}`} +
+
+ + +
+ ) : ( +
+ {`${t['com.affine.settings.workspace.storage.unused-blobs']()} (${unusedBlobs.length})`} +
+ )} +
+ {isLoading ? ( +
+ +
+ ) : unusedBlobs.length === 0 ? ( + + ) : ( + <> +
+ {unusedBlobs.slice(skip, skip + PAGE_SIZE).map(blob => { + const selected = selectedBlobs.includes(blob); + return ( + + selected + ? handleUnselectBlob(blob) + : handleSelectBlob(blob) + } + selected={selected} + /> + ); + })} +
+ { + setPageNum(pageNum); + setSkip(pageNum * PAGE_SIZE); + }} + /> + + )} +
+ + ); +}; diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/index.tsx b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/index.tsx index 3e871a352736e..96468c51ade69 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/index.tsx @@ -6,10 +6,16 @@ import { WorkspaceService } from '@affine/core/modules/workspace'; import { useI18n } from '@affine/i18n'; import { useService } from '@toeverything/infra'; +import { EnableCloudPanel } from '../preference/enable-cloud'; +import { BlobManagementPanel } from './blob-management'; import { DesktopExportPanel } from './export'; import { WorkspaceQuotaPanel } from './workspace-quota'; -export const WorkspaceSettingStorage = () => { +export const WorkspaceSettingStorage = ({ + onCloseSetting, +}: { + onCloseSetting: () => void; +}) => { const t = useI18n(); const workspace = useService(WorkspaceService).workspace; return ( @@ -18,10 +24,18 @@ export const WorkspaceSettingStorage = () => { title={t['Storage']()} subtitle={t['com.affine.settings.workspace.storage.subtitle']()} /> - {workspace.flavour !== 'local' && ( - - - + {workspace.flavour === 'local' ? ( + + ) : ( + <> + + + + + + + + )} {BUILD_CONFIG.isElectron && ( diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/style.css.ts b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/style.css.ts index c448e640c4683..e667e4e3ede4c 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/style.css.ts +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/style.css.ts @@ -28,3 +28,136 @@ globalStyle(`${storageProgressWrapper} .storage-progress-bar-wrapper`, { export const storageProgressBar = style({ height: '100%', }); + +// blob management + +// when no blob is selected +export const blobManagementControls = style({ + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + gap: 16, +}); + +export const spacer = style({ + flexGrow: 1, +}); + +export const blobManagementName = style({ + fontSize: cssVar('fontSm'), + fontWeight: 600, + height: '28px', +}); + +export const blobManagementNameInactive = style([ + blobManagementName, + { + color: cssVarV2('text/secondary'), + }, +]); + +export const blobManagementContainer = style({ + marginTop: '24px', + display: 'flex', + flexDirection: 'column', + gap: '12px', + padding: '12px', + borderRadius: '8px', + background: cssVarV2('layer/background/primary'), + border: `1px solid ${cssVarV2('layer/insideBorder/border')}`, +}); + +export const blobPreviewGrid = style({ + display: 'grid', + gridTemplateColumns: 'repeat(3, minmax(30%, 1fr))', + gap: '12px', +}); + +export const blobCard = style({ + borderRadius: '4px', + overflow: 'hidden', + position: 'relative', +}); + +export const loadingContainer = style({ + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + height: '320px', +}); + +export const empty = style({ + padding: '8px 16px', +}); + +export const blobPreviewContainer = style({ + display: 'flex', + flexDirection: 'column', + justifyContent: 'center', + alignItems: 'center', + gap: 8, +}); + +export const blobPreview = style({ + width: '100%', + overflow: 'hidden', + aspectRatio: '1', + borderRadius: '4px', + padding: 6, + backgroundColor: cssVarV2('layer/background/secondary'), + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + border: `2px solid transparent`, + selectors: { + [`${blobCard}[data-selected="true"] &`]: { + borderColor: cssVarV2('button/primary'), + }, + [`${blobCard}:hover &`]: { + backgroundColor: cssVarV2('layer/background/hoverOverlay'), + }, + }, +}); + +export const blobGridItemCheckbox = style({ + position: 'absolute', + top: 8, + right: 8, + fontSize: 16, + opacity: 0, + selectors: { + [`${blobCard}:hover &`]: { + opacity: 1, + }, + [`${blobCard}[data-selected="true"] &`]: { + opacity: 1, + }, + }, +}); + +export const blobImagePreview = style({ + width: '100%', + height: '100%', + objectFit: 'contain', +}); + +export const unknownBlobIcon = style({}); + +export const blobPreviewFooter = style({ + fontSize: cssVar('fontXs'), + width: '100%', +}); + +export const blobPreviewName = style({ + fontSize: cssVar('fontSm'), + fontWeight: 600, + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + maxWidth: '100%', +}); + +export const blobPreviewInfo = style({ + fontSize: cssVar('fontXs'), + color: cssVarV2('text/secondary'), +}); diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/workspace-quota.tsx b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/workspace-quota.tsx index bca196db92384..b7bef66eee250 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/workspace-quota.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/storage/workspace-quota.tsx @@ -46,7 +46,7 @@ export const StorageProgress = () => { if (loadError) { return Load error; } - return ; + return ; } if (!isTeam) { diff --git a/packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts b/packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts new file mode 100644 index 0000000000000..7143d0452d1b3 --- /dev/null +++ b/packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts @@ -0,0 +1,164 @@ +import type { ListedBlobRecord } from '@affine/nbstore'; +import { + effect, + Entity, + fromPromise, + LiveData, + onComplete, + onStart, +} from '@toeverything/infra'; +import { fileTypeFromBuffer } from 'file-type'; +import { EMPTY, mergeMap, switchMap } from 'rxjs'; + +import type { DocsSearchService } from '../../docs-search'; +import type { WorkspaceService } from '../../workspace'; +import type { WorkspaceFlavoursService } from '../../workspace/services/flavours'; + +interface HydratedBlobRecord extends ListedBlobRecord, Disposable { + url: string; + extension?: string; + type?: string; +} + +export class UnusedBlobs extends Entity { + constructor( + private readonly flavoursService: WorkspaceFlavoursService, + private readonly workspaceService: WorkspaceService, + private readonly docsSearchService: DocsSearchService + ) { + super(); + } + + isLoading$ = new LiveData(false); + unusedBlobs$ = new LiveData([]); + + readonly revalidate = effect( + switchMap(() => + fromPromise(async () => { + return await this.getUnusedBlobs(); + }).pipe( + mergeMap(data => { + this.unusedBlobs$.setValue(data); + return EMPTY; + }), + onStart(() => this.isLoading$.setValue(true)), + onComplete(() => this.isLoading$.setValue(false)) + ) + ) + ); + + private get flavourProvider() { + return this.flavoursService.flavours$.value.find( + f => f.flavour === this.workspaceService.workspace.flavour + ); + } + + private get localFlavourProvider() { + return this.flavoursService.flavours$.value.find( + f => f.flavour === 'local' + ); + } + + async listBlobs() { + const blobs = await this.flavourProvider?.listBlobs( + this.workspaceService.workspace.id + ); + return blobs; + } + + async getBlob(blobKey: string) { + const blob = await this.flavourProvider?.getWorkspaceBlob( + this.workspaceService.workspace.id, + blobKey + ); + return blob; + } + + async deleteBlob(blob: string, permanent: boolean) { + await this.flavourProvider?.deleteBlob( + this.workspaceService.workspace.id, + blob, + permanent + ); + + if (this.localFlavourProvider !== this.flavourProvider) { + await this.localFlavourProvider?.deleteBlob( + this.workspaceService.workspace.id, + blob, + permanent + ); + } + } + + async getUnusedBlobs(abortSignal?: AbortSignal) { + // wait for the indexer to finish + await this.docsSearchService.indexer.status$.waitFor( + status => status.remaining === undefined || status.remaining === 0, + abortSignal + ); + + const [blobs, usedBlobs] = await Promise.all([ + this.listBlobs(), + this.getUsedBlobs(), + ]); + + // ignore the workspace avatar + const workspaceAvatar = this.workspaceService.workspace.avatar$.value; + + return ( + blobs?.filter( + blob => !usedBlobs.includes(blob.key) && blob.key !== workspaceAvatar + ) ?? [] + ); + } + + private async getUsedBlobs(): Promise { + const result = await this.docsSearchService.indexer.blockIndex.aggregate( + { + type: 'boolean', + occur: 'must', + queries: [ + { + type: 'exists', + field: 'blob', + }, + ], + }, + 'blob' + ); + return result.buckets.map(bucket => bucket.key); + } + + async hydrateBlob( + record: ListedBlobRecord, + abortSignal?: AbortSignal + ): Promise { + const blob = await this.getBlob(record.key); + + if (!blob || abortSignal?.aborted) { + return null; + } + + const fileType = await fileTypeFromBuffer(await blob.arrayBuffer()); + + if (abortSignal?.aborted) { + return null; + } + + const url = URL.createObjectURL(new Blob([blob])); + const mime = record.mime || fileType?.mime || 'unknown'; + // todo(@pengx17): the following may not be sufficient + const extension = fileType?.ext; + const type = extension ?? (mime?.startsWith('text/') ? 'txt' : 'unknown'); + return { + ...record, + url, + extension, + type, + mime, + [Symbol.dispose]: () => { + URL.revokeObjectURL(url); + }, + }; + } +} diff --git a/packages/frontend/core/src/modules/blob-management/index.ts b/packages/frontend/core/src/modules/blob-management/index.ts new file mode 100644 index 0000000000000..5fd318e210d63 --- /dev/null +++ b/packages/frontend/core/src/modules/blob-management/index.ts @@ -0,0 +1,18 @@ +import { type Framework } from '@toeverything/infra'; + +import { DocsSearchService } from '../docs-search'; +import { WorkspaceScope, WorkspaceService } from '../workspace'; +import { WorkspaceFlavoursService } from '../workspace/services/flavours'; +import { UnusedBlobs } from './entity/unused-blobs'; +import { BlobManagementService } from './services'; + +export function configureBlobManagementModule(framework: Framework) { + framework + .scope(WorkspaceScope) + .entity(UnusedBlobs, [ + WorkspaceFlavoursService, + WorkspaceService, + DocsSearchService, + ]) + .service(BlobManagementService); +} diff --git a/packages/frontend/core/src/modules/blob-management/services/index.ts b/packages/frontend/core/src/modules/blob-management/services/index.ts new file mode 100644 index 0000000000000..17224501c1b7f --- /dev/null +++ b/packages/frontend/core/src/modules/blob-management/services/index.ts @@ -0,0 +1,11 @@ +import { Service } from '@toeverything/infra'; + +import { UnusedBlobs } from '../entity/unused-blobs'; + +export class BlobManagementService extends Service { + constructor() { + super(); + } + + unusedBlobs = this.framework.createEntity(UnusedBlobs); +} diff --git a/packages/frontend/core/src/modules/index.ts b/packages/frontend/core/src/modules/index.ts index db0d34cfb6930..296dbd6f893f0 100644 --- a/packages/frontend/core/src/modules/index.ts +++ b/packages/frontend/core/src/modules/index.ts @@ -7,6 +7,7 @@ import { } from './ai-button'; import { configureAppSidebarModule } from './app-sidebar'; import { configAtMenuConfigModule } from './at-menu-config'; +import { configureBlobManagementModule } from './blob-management'; import { configureCloudModule } from './cloud'; import { configureCollectionModule } from './collection'; import { configureWorkspaceDBModule } from './db'; @@ -98,4 +99,5 @@ export function configureCommonModules(framework: Framework) { configureAINetworkSearchModule(framework); configureAIButtonModule(framework); configureTemplateDocModule(framework); + configureBlobManagementModule(framework); } diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts index a899d4955ee95..bd2481252304e 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts @@ -5,7 +5,11 @@ import { getWorkspaceInfoQuery, getWorkspacesQuery, } from '@affine/graphql'; -import type { BlobStorage, DocStorage } from '@affine/nbstore'; +import type { + BlobStorage, + DocStorage, + ListedBlobRecord, +} from '@affine/nbstore'; import { CloudBlobStorage, StaticCloudDocStorage } from '@affine/nbstore/cloud'; import { IndexedDBBlobStorage, @@ -364,6 +368,26 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { return new Blob([cloudBlob.data], { type: cloudBlob.mime }); } + async listBlobs(id: string): Promise { + const cloudStorage = new CloudBlobStorage({ + id, + serverBaseUrl: this.server.serverMetadata.baseUrl, + }); + return cloudStorage.list(); + } + + async deleteBlob( + id: string, + blob: string, + permanent: boolean + ): Promise { + const cloudStorage = new CloudBlobStorage({ + id, + serverBaseUrl: this.server.serverMetadata.baseUrl, + }); + await cloudStorage.delete(blob, permanent); + } + onWorkspaceInitialized(workspace: Workspace): void { // bind the workspace to the affine cloud server workspace.scope.get(WorkspaceServerService).bindServer(this.server); diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/local.ts b/packages/frontend/core/src/modules/workspace-engine/impls/local.ts index 8eaec84c5b7cb..93a63c5834561 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/local.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/local.ts @@ -2,6 +2,7 @@ import { DebugLogger } from '@affine/debug'; import { type BlobStorage, type DocStorage, + type ListedBlobRecord, universalId, } from '@affine/nbstore'; import { @@ -276,6 +277,33 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider { return blob ? new Blob([blob.data], { type: blob.mime }) : null; } + async listBlobs(id: string): Promise { + const storage = new this.BlobStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + }); + storage.connection.connect(); + await storage.connection.waitForConnected(); + + return storage.list(); + } + + async deleteBlob( + id: string, + blob: string, + permanent: boolean + ): Promise { + const storage = new this.BlobStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + }); + storage.connection.connect(); + await storage.connection.waitForConnected(); + await storage.delete(blob, permanent); + } + getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions { return { local: { diff --git a/packages/frontend/core/src/modules/workspace/providers/flavour.ts b/packages/frontend/core/src/modules/workspace/providers/flavour.ts index aef6f46bc25cf..f08e3a58c9fd7 100644 --- a/packages/frontend/core/src/modules/workspace/providers/flavour.ts +++ b/packages/frontend/core/src/modules/workspace/providers/flavour.ts @@ -1,4 +1,8 @@ -import type { BlobStorage, DocStorage } from '@affine/nbstore'; +import type { + BlobStorage, + DocStorage, + ListedBlobRecord, +} from '@affine/nbstore'; import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; import type { Workspace as BSWorkspace } from '@blocksuite/affine/store'; import { createIdentifier, type LiveData } from '@toeverything/infra'; @@ -41,6 +45,14 @@ export interface WorkspaceFlavourProvider { getWorkspaceBlob(id: string, blob: string): Promise; + listBlobs(workspaceId: string): Promise; + + deleteBlob( + workspaceId: string, + blob: string, + permanent: boolean + ): Promise; + getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions; onWorkspaceInitialized?(workspace: Workspace): void; diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index e2c626565aa94..19bfd8b0b66ff 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -1659,5 +1659,10 @@ "com.affine.settings.workspace.template.page": "New doc with template", "com.affine.settings.workspace.template.page-desc": "New docs will use the specified template, ignoring default settings.", "com.affine.settings.workspace.template.page-select": "Template for new doc", - "com.affine.settings.workspace.template.remove": "Remove template" + "com.affine.settings.workspace.template.remove": "Remove template", + "com.affine.settings.workspace.storage.unused-blobs": "Unused blobs", + "com.affine.settings.workspace.storage.unused-blobs.empty": "No unused blobs", + "com.affine.settings.workspace.storage.unused-blobs.selected": "Selected", + "com.affine.settings.workspace.storage.unused-blobs.delete.title": "Delete blob files", + "com.affine.settings.workspace.storage.unused-blobs.delete.warning": "Are you sure you want to delete these blob files? This action cannot be undone. Make sure you no longer need them before proceeding." } diff --git a/tests/affine-cloud/e2e/storage.spec.ts b/tests/affine-cloud/e2e/storage.spec.ts new file mode 100644 index 0000000000000..7a452aefc7969 --- /dev/null +++ b/tests/affine-cloud/e2e/storage.spec.ts @@ -0,0 +1,61 @@ +import { test } from '@affine-test/kit/playwright'; +import { + createRandomUser, + deleteUser, + enableCloudWorkspace, + loginUser, +} from '@affine-test/kit/utils/cloud'; +import { clickSideBarAllPageButton } from '@affine-test/kit/utils/sidebar'; +import { expect } from '@playwright/test'; + +let user: { + id: string; + name: string; + email: string; + password: string; +}; + +test.beforeEach(async ({ page }) => { + user = await createRandomUser(); + await loginUser(page, user); +}); + +test.afterEach(async () => { + // if you want to keep the user in the database for debugging, + // comment this line + await deleteUser(user.email); +}); + +test('should show blob management dialog', async ({ page }) => { + await enableCloudWorkspace(page); + + await clickSideBarAllPageButton(page); + + // delete the welcome page ('Write, draw, plan all at once.') + await page + .getByTestId('page-list-item') + .filter({ + has: page.getByText('Write, draw, plan all at once.'), + }) + .getByTestId('page-list-operation-button') + .click(); + const deleteBtn = page.getByTestId('move-to-trash'); + await deleteBtn.click(); + await expect(page.getByText('Delete doc?')).toBeVisible(); + await page.getByRole('button', { name: 'Delete' }).click(); + + await page.getByTestId('slider-bar-workspace-setting-button').click(); + await expect(page.getByTestId('setting-modal')).toBeVisible(); + await page.getByTestId('workspace-setting:storage').click(); + await expect(page.getByTestId('blob-preview-card')).toHaveCount(3); + await expect(page.getByText('Unused blobs (3)')).toBeVisible(); + + await page.getByTestId('blob-preview-card').nth(0).click(); + await expect(page.getByText('1 Selected')).toBeVisible(); + + await page.getByRole('button', { name: 'Delete' }).click(); + await expect(page.getByText('Delete blob files')).toBeVisible(); + await page.getByRole('button', { name: 'Delete' }).click(); + + await expect(page.getByText('Unused blobs (2)')).toBeVisible(); +}); diff --git a/tests/affine-desktop/e2e/workspace.spec.ts b/tests/affine-desktop/e2e/workspace.spec.ts index 3cdb671f61327..782786bdb8036 100644 --- a/tests/affine-desktop/e2e/workspace.spec.ts +++ b/tests/affine-desktop/e2e/workspace.spec.ts @@ -95,6 +95,8 @@ test('export then add', async ({ page, appInfo, workspace }) => { // check its name is correct await expect(page.getByTestId('workspace-name')).toHaveText(newWorkspaceName); + await page.waitForTimeout(1000); + // find button which has the title "test1" await page.getByText('test1').click();