feat(server): use doc service

.github/workflows/build-images.yml

@@ -28,7 +28,7 @@ jobs:
           extra-flags: workspaces focus @affine/server
       - name: Build Server
         run: |
-          rm -rf packages/backend/server/src/__tests__
+          find packages/backend/server -type d -name "__tests__" -exec rm -rf {} +
           yarn workspace @affine/server build
       - name: Upload server dist
         uses: actions/upload-artifact@v4

packages/backend/server/src/__tests__/doc/cron.spec.ts

@@ -1,67 +1,39 @@
-import { mock } from 'node:test';
-
 import { ScheduleModule } from '@nestjs/schedule';
-import { TestingModule } from '@nestjs/testing';
 import { PrismaClient } from '@prisma/client';
-import test from 'ava';
-import * as Sinon from 'sinon';
+import ava, { TestFn } from 'ava';
 
-import { Config } from '../../base/config';
 import { DocStorageModule } from '../../core/doc';
 import { DocStorageCronJob } from '../../core/doc/job';
-import { createTestingModule } from '../utils';
+import { createTestingModule, type TestingModule } from '../utils';
+
+interface Context {
+  module: TestingModule;
+  db: PrismaClient;
+  cronJob: DocStorageCronJob;
+}
 
-let m: TestingModule;
-let timer: Sinon.SinonFakeTimers;
-let db: PrismaClient;
+const test = ava as TestFn<Context>;
 
 // cleanup database before each test
-test.before(async () => {
-  timer = Sinon.useFakeTimers({
-    toFake: ['setInterval'],
-  });
-  m = await createTestingModule({
+test.before(async t => {
+  t.context.module = await createTestingModule({
     imports: [ScheduleModule.forRoot(), DocStorageModule],
   });
 
-  db = m.get(PrismaClient);
+  t.context.db = t.context.module.get(PrismaClient);
+  t.context.cronJob = t.context.module.get(DocStorageCronJob);
 });
 
-test.after.always(async () => {
-  await m.close();
-  timer.restore();
+test.beforeEach(async t => {
+  await t.context.module.initTestingDB();
 });
 
-test('should poll when intervel due', async t => {
-  const manager = m.get(DocStorageCronJob);
-  const interval = m.get(Config).doc.manager.updatePollInterval;
-
-  let resolve: any;
-  const fake = mock.method(manager, 'autoMergePendingDocUpdates', () => {
-    return new Promise(_resolve => {
-      resolve = _resolve;
-    });
-  });
-
-  timer.tick(interval);
-  t.is(fake.mock.callCount(), 1);
-
-  // busy
-  timer.tick(interval);
-  // @ts-expect-error private member
-  t.is(manager.busy, true);
-  t.is(fake.mock.callCount(), 1);
-
-  resolve();
-  await timer.tickAsync(1);
-
-  // @ts-expect-error private member
-  t.is(manager.busy, false);
-  timer.tick(interval);
-  t.is(fake.mock.callCount(), 2);
+test.after.always(async t => {
+  await t.context.module.close();
 });
 
 test('should be able to cleanup expired history', async t => {
+  const { db } = t.context;
   const timestamp = Date.now();
 
   // insert expired data
@@ -93,7 +65,7 @@ test('should be able to cleanup expired history', async t => {
   let count = await db.snapshotHistory.count();
   t.is(count, 20);
 
-  await m.get(DocStorageCronJob).cleanupExpiredHistory();
+  await t.context.cronJob.cleanupExpiredHistory();
 
   count = await db.snapshotHistory.count();
   t.is(count, 10);

packages/backend/server/src/__tests__/utils/utils.ts

@@ -1,8 +1,4 @@
-import {
-  ConsoleLogger,
-  INestApplication,
-  ModuleMetadata,
-} from '@nestjs/common';
+import { INestApplication, LogLevel, ModuleMetadata } from '@nestjs/common';
 import { APP_GUARD, ModuleRef } from '@nestjs/core';
 import { Query, Resolver } from '@nestjs/graphql';
 import {
@@ -17,12 +13,15 @@ import type { Response } from 'supertest';
 import supertest from 'supertest';
 
 import { AppModule, FunctionalityModules } from '../../app.module';
-import { GlobalExceptionFilter, Runtime } from '../../base';
+import { AFFiNELogger, GlobalExceptionFilter, Runtime } from '../../base';
 import { GqlModule } from '../../base/graphql';
 import { AuthGuard, AuthModule } from '../../core/auth';
 import { RefreshFeatures0001 } from '../../data/migrations/0001-refresh-features';
 import { ModelsModule } from '../../models';
 
+const TEST_LOG_LEVEL: LogLevel =
+  (process.env.TEST_LOG_LEVEL as LogLevel) ?? 'fatal';
+
 async function flushDB(client: PrismaClient) {
   const result: { tablename: string }[] =
     await client.$queryRaw`SELECT tablename
@@ -39,7 +38,7 @@ async function flushDB(client: PrismaClient) {
   );
 }
 
-interface TestingModuleMeatdata extends ModuleMetadata {
+interface TestingModuleMetadata extends ModuleMetadata {
   tapModule?(m: TestingModuleBuilder): void;
   tapApp?(app: INestApplication): void;
 }
@@ -85,7 +84,7 @@ class MockResolver {
 }
 
 export async function createTestingModule(
-  moduleDef: TestingModuleMeatdata = {},
+  moduleDef: TestingModuleMetadata = {},
   autoInitialize = true
 ): Promise<TestingModule> {
   // setting up
@@ -127,7 +126,7 @@ export async function createTestingModule(
     // can't tolerate the noisy logs
     // @ts-expect-error private
     m.applyLogger({
-      logger: ['fatal'],
+      logger: [TEST_LOG_LEVEL],
     });
     const runtime = m.get(Runtime);
     // by pass password min length validation
@@ -146,7 +145,7 @@ export async function createTestingModule(
 }
 
 export async function createTestingApp(
-  moduleDef: TestingModuleMeatdata = {}
+  moduleDef: TestingModuleMetadata = {}
 ): Promise<{ module: TestingModule; app: TestingApp }> {
   const m = await createTestingModule(moduleDef, false);
 
@@ -155,9 +154,9 @@ export async function createTestingApp(
     bodyParser: true,
     rawBody: true,
   }) as TestingApp;
-  const logger = new ConsoleLogger();
+  const logger = new AFFiNELogger();
 
-  logger.setLogLevels(['fatal']);
+  logger.setLogLevels([TEST_LOG_LEVEL]);
   app.useLogger(logger);
 
   app.useGlobalFilters(new GlobalExceptionFilter(app.getHttpAdapter()));

packages/backend/server/src/app.module.ts

@@ -1,5 +1,3 @@
-import { randomUUID } from 'node:crypto';
-
 import {
   DynamicModule,
   ForwardReference,
@@ -15,7 +13,7 @@ import { get } from 'lodash-es';
 import { ClsModule } from 'nestjs-cls';
 
 import { AppController } from './app.controller';
-import { getOptionalModuleMetadata } from './base';
+import { genRequestId, getOptionalModuleMetadata } from './base';
 import { CacheModule } from './base/cache';
 import { AFFiNEConfig, ConfigModule, mergeConfigOverride } from './base/config';
 import { ErrorModule } from './base/error';
@@ -59,7 +57,7 @@ export const FunctionalityModules = [
       generateId: true,
       idGenerator(req: Request) {
         // make every request has a unique id to tracing
-        return req.get('x-rpc-trace-id') ?? `req-${randomUUID()}`;
+        return req.get('x-rpc-trace-id') ?? genRequestId('req');
       },
       setup(cls, _req, res: Response) {
         res.setHeader('X-Request-Id', cls.getId());
@@ -72,7 +70,7 @@ export const FunctionalityModules = [
       generateId: true,
       idGenerator() {
         // make every request has a unique id to tracing
-        return `ws-${randomUUID()}`;
+        return genRequestId('ws');
       },
     },
     plugins: [
@@ -200,6 +198,12 @@ export function buildAppModule() {
     .use(...FunctionalityModules)
     .use(ModelsModule)
 
+    // enable schedule module on graphql server and doc service
+    .useIf(
+      config => config.flavor.graphql || config.flavor.doc,
+      ScheduleModule.forRoot()
+    )
+
     // auth
     .use(UserModule, AuthModule, PermissionModule)
 
@@ -212,7 +216,6 @@ export function buildAppModule() {
     // graphql server only
     .useIf(
       config => config.flavor.graphql,
-      ScheduleModule.forRoot(),
       GqlModule,
       StorageModule,
       ServerConfigModule,

packages/backend/server/src/base/event/eventbus.ts

@@ -1,5 +1,3 @@
-import { randomUUID } from 'node:crypto';
-
 import {
   applyDecorators,
   Injectable,
@@ -21,6 +19,7 @@ import { CLS_ID, ClsService } from 'nestjs-cls';
 import type { Server, Socket } from 'socket.io';
 
 import { CallMetric } from '../metrics';
+import { genRequestId } from '../utils';
 import type { EventName } from './def';
 
 const EventHandlerWrapper = (event: EventName): MethodDecorator => {
@@ -94,7 +93,7 @@ export class EventBus implements OnGatewayConnection, OnApplicationBootstrap {
           // to internal event system
           this.server?.on(event, (payload, requestId?: string) => {
             this.cls.run(() => {
-              requestId = requestId ?? `server_event-${randomUUID()}`;
+              requestId = requestId ?? genRequestId('se');
               this.cls.set(CLS_ID, requestId);
               this.logger.log(`Server Event: ${event} (Received)`);
               this.emit(event, payload);

packages/backend/server/src/base/utils/request.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import { IncomingMessage } from 'node:http';
 
 import type { ArgumentsHost, ExecutionContext } from '@nestjs/common';
@@ -77,3 +78,18 @@ export function parseCookies(
     {} as Record<string, string>
   );
 }
+
+/**
+ * Request type
+ *
+ * @description
+ * - `req`: http request
+ * - `ws`: websocket request
+ * - `se`: server event
+ * - `job`: cron job
+ */
+export type RequestType = 'req' | 'ws' | 'se' | 'job';
+
+export function genRequestId(type: RequestType) {
+  return `${AFFiNE.flavor.type}:${type}-${randomUUID()}`;
+}

packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts

@@ -0,0 +1,85 @@
+import { randomUUID } from 'node:crypto';
+
+import { User, Workspace } from '@prisma/client';
+import ava, { TestFn } from 'ava';
+import request from 'supertest';
+import { Doc as YDoc } from 'yjs';
+
+import { createTestingApp, type TestingApp } from '../../../__tests__/utils';
+import { AppModule } from '../../../app.module';
+import { Config } from '../../../base';
+import { ConfigModule } from '../../../base/config';
+import { Models } from '../../../models';
+import { PgWorkspaceDocStorageAdapter } from '../../doc';
+import { PermissionService } from '../../permission';
+
+const test = ava as TestFn<{
+  models: Models;
+  app: TestingApp;
+  config: Config;
+  adapter: PgWorkspaceDocStorageAdapter;
+  permission: PermissionService;
+}>;
+
+test.before(async t => {
+  const { app } = await createTestingApp({
+    imports: [
+      ConfigModule.forRoot({
+        flavor: {
+          doc: false,
+        },
+        docService: {
+          endpoint: '',
+        },
+      }),
+      AppModule,
+    ],
+  });
+
+  t.context.models = app.get(Models);
+  t.context.config = app.get(Config);
+  t.context.adapter = app.get(PgWorkspaceDocStorageAdapter);
+  t.context.permission = app.get(PermissionService);
+  t.context.app = app;
+});
+
+let user: User;
+let workspace: Workspace;
+
+test.beforeEach(async t => {
+  t.context.config.docService.endpoint = t.context.app.getHttpServerUrl();
+  await t.context.app.initTestingDB();
+  user = await t.context.models.user.create({
+    email: '[email protected]',
+  });
+  workspace = await t.context.models.workspace.create(user.id);
+});
+
+test.after.always(async t => {
+  await t.context.app.close();
+});
+
+test('should render page success', async t => {
+  const docId = randomUUID();
+  const { app, adapter, permission } = t.context;
+
+  const doc = new YDoc();
+  const text = doc.getText('content');
+  const updates: Buffer[] = [];
+
+  doc.on('update', update => {
+    updates.push(Buffer.from(update));
+  });
+
+  text.insert(0, 'hello');
+  text.insert(5, 'world');
+  text.insert(5, ' ');
+
+  await adapter.pushDocUpdates(workspace.id, docId, updates, user.id);
+  await permission.publishPage(workspace.id, docId);
+
+  await request(app.getHttpServer())
+    .get(`/workspace/${workspace.id}/${docId}`)
+    .expect(200);
+  t.pass();
+});