diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs index e5a35a18bf070..b067265248d9e 100644 --- a/.github/actions/deploy/deploy.mjs +++ b/.github/actions/deploy/deploy.mjs @@ -47,18 +47,21 @@ const replicaConfig = { graphql: Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 3, sync: Number(process.env.PRODUCTION_SYNC_REPLICA) || 3, renderer: Number(process.env.PRODUCTION_RENDERER_REPLICA) || 3, + doc: Number(process.env.PRODUCTION_DOC_REPLICA) || 3, }, beta: { web: 2, graphql: Number(process.env.BETA_GRAPHQL_REPLICA) || 2, sync: Number(process.env.BETA_SYNC_REPLICA) || 2, renderer: Number(process.env.BETA_RENDERER_REPLICA) || 2, + doc: Number(process.env.BETA_DOC_REPLICA) || 2, }, canary: { web: 2, graphql: 2, sync: 2, renderer: 2, + doc: 2, }, }; @@ -67,12 +70,14 @@ const cpuConfig = { web: '300m', graphql: '1', sync: '1', + doc: '1', renderer: '300m', }, canary: { web: '300m', graphql: '1', sync: '1', + doc: '1', renderer: '300m', }, }; @@ -111,6 +116,7 @@ const createHelmCommand = ({ isDryRun }) => { `--set web.resources.requests.cpu="${cpu.web}"`, `--set graphql.resources.requests.cpu="${cpu.graphql}"`, `--set sync.resources.requests.cpu="${cpu.sync}"`, + `--set doc.resources.requests.cpu="${cpu.doc}"`, ] : []; @@ -168,6 +174,9 @@ const createHelmCommand = ({ isDryRun }) => { `--set-string renderer.image.tag="${imageTag}"`, `--set renderer.app.host=${host}`, `--set renderer.replicaCount=${replica.renderer}`, + `--set-string doc.image.tag="${imageTag}"`, + `--set doc.app.host=${host}`, + `--set doc.replicaCount=${replica.doc}`, ...serviceAnnotations, ...resources, `--timeout 10m`, diff --git a/.github/helm/affine/charts/doc/Chart.yaml b/.github/helm/affine/charts/doc/Chart.yaml new file mode 100644 index 0000000000000..df6cd62685742 --- /dev/null +++ b/.github/helm/affine/charts/doc/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: doc +description: AFFiNE doc server +type: application +version: 0.0.0 +appVersion: "0.20.0" +dependencies: + - name: gcloud-sql-proxy + version: 0.0.0 + repository: "file://../gcloud-sql-proxy" + condition: .global.database.gcloud.enabled diff --git a/.github/helm/affine/charts/doc/templates/NOTES.txt b/.github/helm/affine/charts/doc/templates/NOTES.txt new file mode 100644 index 0000000000000..1e28e8e0847e1 --- /dev/null +++ b/.github/helm/affine/charts/doc/templates/NOTES.txt @@ -0,0 +1,16 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "doc.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "doc.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "doc.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "doc.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/.github/helm/affine/charts/doc/templates/_helpers.tpl b/.github/helm/affine/charts/doc/templates/_helpers.tpl new file mode 100644 index 0000000000000..7741afd2b3370 --- /dev/null +++ b/.github/helm/affine/charts/doc/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "doc.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "doc.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "doc.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "doc.labels" -}} +helm.sh/chart: {{ include "doc.chart" . }} +{{ include "doc.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +monitoring: enabled +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "doc.selectorLabels" -}} +app.kubernetes.io/name: {{ include "doc.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "doc.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "doc.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/doc/templates/deployment.yaml b/.github/helm/affine/charts/doc/templates/deployment.yaml new file mode 100644 index 0000000000000..b28870be786ac --- /dev/null +++ b/.github/helm/affine/charts/doc/templates/deployment.yaml @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "doc.fullname" . }} + labels: + {{- include "doc.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "doc.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "doc.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "doc.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AFFINE_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.global.secret.secretName }}" + key: key + - name: NODE_ENV + value: "{{ .Values.env }}" + - name: NODE_OPTIONS + value: "--max-old-space-size=4096" + - name: NO_COLOR + value: "1" + - name: DEPLOYMENT_TYPE + value: "affine" + - name: SERVER_FLAVOR + value: "doc" + - name: AFFINE_ENV + value: "{{ .Release.Namespace }}" + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-postgresql + key: postgres-password + - name: DATABASE_URL + value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} + - name: REDIS_SERVER_ENABLED + value: "true" + - name: REDIS_SERVER_HOST + value: "{{ .Values.global.redis.host }}" + - name: REDIS_SERVER_PORT + value: "{{ .Values.global.redis.port }}" + - name: REDIS_SERVER_USER + value: "{{ .Values.global.redis.username }}" + - name: REDIS_SERVER_PASSWORD + valueFrom: + secretKeyRef: + name: redis + key: redis-password + - name: REDIS_SERVER_DATABASE + value: "{{ .Values.global.redis.database }}" + - name: AFFINE_SERVER_PORT + value: "{{ .Values.service.port }}" + - name: AFFINE_SERVER_SUB_PATH + value: "{{ .Values.app.path }}" + - name: AFFINE_SERVER_HOST + value: "{{ .Values.app.host }}" + - name: AFFINE_SERVER_HTTPS + value: "{{ .Values.app.https }}" + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /info + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + readinessProbe: + httpGet: + path: /info + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/.github/helm/affine/charts/doc/templates/service.yaml b/.github/helm/affine/charts/doc/templates/service.yaml new file mode 100644 index 0000000000000..29441653218f6 --- /dev/null +++ b/.github/helm/affine/charts/doc/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "doc.fullname" . }} + labels: + {{- include "doc.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "doc.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/doc/templates/serviceaccount.yaml b/.github/helm/affine/charts/doc/templates/serviceaccount.yaml new file mode 100644 index 0000000000000..3b973fc989e9e --- /dev/null +++ b/.github/helm/affine/charts/doc/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "doc.serviceAccountName" . }} + labels: + {{- include "doc.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/doc/templates/tests/test-connection.yaml b/.github/helm/affine/charts/doc/templates/tests/test-connection.yaml new file mode 100644 index 0000000000000..3df384e693c73 --- /dev/null +++ b/.github/helm/affine/charts/doc/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "doc.fullname" . }}-test-connection" + labels: + {{- include "doc.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "doc.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/.github/helm/affine/charts/doc/values.yaml b/.github/helm/affine/charts/doc/values.yaml new file mode 100644 index 0000000000000..eeb0496cb882c --- /dev/null +++ b/.github/helm/affine/charts/doc/values.yaml @@ -0,0 +1,38 @@ +replicaCount: 1 +image: + repository: ghcr.io/toeverything/affine-graphql + pullPolicy: IfNotPresent + tag: '' + +imagePullSecrets: [] +nameOverride: '' +fullnameOverride: '' +# map to NODE_ENV environment variable +env: 'production' +app: + # AFFINE_SERVER_SUB_PATH + path: '' + # AFFINE_SERVER_HOST + host: '0.0.0.0' + https: true +serviceAccount: + create: true + annotations: {} + name: 'affine-doc' + +podAnnotations: {} + +podSecurityContext: + fsGroup: 2000 + +resources: + requests: + cpu: '2' + memory: 4Gi + +probe: + initialDelaySeconds: 20 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index aaa79e0dcfe52..e9267e9d4aa9c 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -118,6 +118,8 @@ spec: key: stripeWebhookKey - name: DOC_MERGE_INTERVAL value: "{{ .Values.app.doc.mergeInterval }}" + - name: DOC_SERVICE_ENDPOINT + value: "{{ .Values.global.docService.endpoint }}" {{ if .Values.app.experimental.enableJwstCodec }} - name: DOC_MERGE_USE_JWST_CODEC value: "true" diff --git a/.github/helm/affine/charts/renderer/templates/deployment.yaml b/.github/helm/affine/charts/renderer/templates/deployment.yaml index 6a6edc380aa3a..01fe5df9fc14d 100644 --- a/.github/helm/affine/charts/renderer/templates/deployment.yaml +++ b/.github/helm/affine/charts/renderer/templates/deployment.yaml @@ -94,6 +94,8 @@ spec: name: "{{ .Values.global.objectStorage.r2.secretName }}" key: secretAccessKey {{ end }} + - name: DOC_SERVICE_ENDPOINT + value: "{{ .Values.global.docService.endpoint }}" ports: - name: http containerPort: {{ .Values.service.port }} diff --git a/.github/helm/affine/charts/sync/templates/deployment.yaml b/.github/helm/affine/charts/sync/templates/deployment.yaml index 19c62df680b6a..77cfea22b3692 100644 --- a/.github/helm/affine/charts/sync/templates/deployment.yaml +++ b/.github/helm/affine/charts/sync/templates/deployment.yaml @@ -73,6 +73,8 @@ spec: value: "{{ .Values.service.port }}" - name: AFFINE_SERVER_HOST value: "{{ .Values.app.host }}" + - name: DOC_SERVICE_ENDPOINT + value: "{{ .Values.global.docService.endpoint }}" ports: - name: http containerPort: {{ .Values.service.port }} diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml index 1f2f9e216d1a8..1615a825a7480 100644 --- a/.github/helm/affine/values.yaml +++ b/.github/helm/affine/values.yaml @@ -39,6 +39,8 @@ global: secretAccessKey: '' gke: enabled: true + docService: + endpoint: 'http://affine-doc:3020' graphql: service: @@ -61,6 +63,13 @@ renderer: annotations: cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' +doc: + service: + type: ClusterIP + port: 3020 + annotations: + cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' + web: service: type: ClusterIP diff --git a/packages/backend/server/src/__tests__/app/doc.e2e.ts b/packages/backend/server/src/__tests__/app/doc.e2e.ts new file mode 100644 index 0000000000000..127cb57c30ffd --- /dev/null +++ b/packages/backend/server/src/__tests__/app/doc.e2e.ts @@ -0,0 +1,40 @@ +import type { INestApplication } from '@nestjs/common'; +import type { TestFn } from 'ava'; +import ava from 'ava'; +import request from 'supertest'; + +import { buildAppModule } from '../../app.module'; +import { createTestingApp } from '../utils'; + +const test = ava as TestFn<{ + app: INestApplication; +}>; + +test.before('start app', async t => { + // @ts-expect-error override + AFFiNE.flavor = { + type: 'doc', + allinone: false, + graphql: false, + sync: false, + renderer: false, + doc: true, + } satisfies typeof AFFiNE.flavor; + const { app } = await createTestingApp({ + imports: [buildAppModule()], + }); + + t.context.app = app; +}); + +test.after.always(async t => { + await t.context.app.close(); +}); + +test('should init app', async t => { + const res = await request(t.context.app.getHttpServer()) + .get('/info') + .expect(200); + + t.is(res.body.flavor, 'doc'); +}); diff --git a/packages/backend/server/src/__tests__/app/graphql.e2e.ts b/packages/backend/server/src/__tests__/app/graphql.e2e.ts index 29c6008f40714..573538374f2ce 100644 --- a/packages/backend/server/src/__tests__/app/graphql.e2e.ts +++ b/packages/backend/server/src/__tests__/app/graphql.e2e.ts @@ -20,6 +20,7 @@ test.before('start app', async t => { graphql: true, sync: false, renderer: false, + doc: false, } satisfies typeof AFFiNE.flavor; const { app } = await createTestingApp({ imports: [buildAppModule()], diff --git a/packages/backend/server/src/__tests__/app/renderer.e2e.ts b/packages/backend/server/src/__tests__/app/renderer.e2e.ts index 1c80c0ccdc357..9a558b6a5ba8a 100644 --- a/packages/backend/server/src/__tests__/app/renderer.e2e.ts +++ b/packages/backend/server/src/__tests__/app/renderer.e2e.ts @@ -18,6 +18,7 @@ test.before('start app', async t => { graphql: false, sync: false, renderer: true, + doc: false, } satisfies typeof AFFiNE.flavor; const { app } = await createTestingApp({ imports: [buildAppModule()], diff --git a/packages/backend/server/src/__tests__/app/sync.e2e.ts b/packages/backend/server/src/__tests__/app/sync.e2e.ts index 471a548cff0df..2a0413cd26dc4 100644 --- a/packages/backend/server/src/__tests__/app/sync.e2e.ts +++ b/packages/backend/server/src/__tests__/app/sync.e2e.ts @@ -18,6 +18,7 @@ test.before('start app', async t => { graphql: false, sync: true, renderer: false, + doc: false, } satisfies typeof AFFiNE.flavor; const { app } = await createTestingApp({ imports: [buildAppModule()], diff --git a/packages/backend/server/src/__tests__/utils/utils.ts b/packages/backend/server/src/__tests__/utils/utils.ts index 1b0143d16e466..1064b43fafc35 100644 --- a/packages/backend/server/src/__tests__/utils/utils.ts +++ b/packages/backend/server/src/__tests__/utils/utils.ts @@ -58,6 +58,8 @@ export type TestingModule = BaseTestingModule & { export type TestingApp = INestApplication & { initTestingDB(): Promise; [Symbol.asyncDispose](): Promise; + // get the url of the http server, e.g. http://localhost:random-port + getHttpServerUrl(): string; }; function dedupeModules(modules: NonNullable) { @@ -180,6 +182,15 @@ export async function createTestingApp( await m[Symbol.asyncDispose](); await app.close(); }; + + app.getHttpServerUrl = () => { + const server = app.getHttpServer(); + if (!server.address()) { + server.listen(); + } + return `http://localhost:${server.address().port}`; + }; + return { module: m, app: app, diff --git a/packages/backend/server/src/app.module.ts b/packages/backend/server/src/app.module.ts index 8af87b6328dd2..ac82aad904a50 100644 --- a/packages/backend/server/src/app.module.ts +++ b/packages/backend/server/src/app.module.ts @@ -10,7 +10,7 @@ import { ScheduleModule } from '@nestjs/schedule'; import { ClsPluginTransactional } from '@nestjs-cls/transactional'; import { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-prisma'; import { PrismaClient } from '@prisma/client'; -import { Response } from 'express'; +import { Request, Response } from 'express'; import { get } from 'lodash-es'; import { ClsModule } from 'nestjs-cls'; @@ -36,6 +36,7 @@ import { AuthModule } from './core/auth'; import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config'; import { DocStorageModule } from './core/doc'; import { DocRendererModule } from './core/doc-renderer'; +import { DocServiceModule } from './core/doc-service'; import { FeatureModule } from './core/features'; import { PermissionModule } from './core/permission'; import { QuotaModule } from './core/quota'; @@ -56,9 +57,9 @@ export const FunctionalityModules = [ middleware: { mount: true, generateId: true, - idGenerator() { + idGenerator(req: Request) { // make every request has a unique id to tracing - return `req-${randomUUID()}`; + return req.get('x-rpc-trace-id') ?? `req-${randomUUID()}`; }, setup(cls, _req, res: Response) { res.setHeader('X-Request-Id', cls.getId()); @@ -203,7 +204,7 @@ export function buildAppModule() { .use(UserModule, AuthModule, PermissionModule) // business modules - .use(FeatureModule, QuotaModule, DocStorageModule) + .use(FeatureModule, QuotaModule, DocStorageModule, DocServiceModule) // sync server only .useIf(config => config.flavor.sync, SyncModule) diff --git a/packages/backend/server/src/base/config/def.ts b/packages/backend/server/src/base/config/def.ts index b70f6ea5b06b2..6e221639804a5 100644 --- a/packages/backend/server/src/base/config/def.ts +++ b/packages/backend/server/src/base/config/def.ts @@ -2,7 +2,7 @@ import type { LeafPaths } from '../utils/types'; import { AppStartupConfig } from './types'; export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean'; -export type ServerFlavor = 'allinone' | 'graphql' | 'sync' | 'renderer'; +export type ServerFlavor = 'allinone' | 'graphql' | 'sync' | 'renderer' | 'doc'; export type AFFINE_ENV = 'dev' | 'beta' | 'production'; export type NODE_ENV = 'development' | 'test' | 'production' | 'script'; @@ -41,9 +41,9 @@ export type AFFiNEConfig = PreDefinedAFFiNEConfig & AppPluginsConfig; declare global { - // eslint-disable-next-line @typescript-eslint/no-namespace + // oxlint-disable-next-line @typescript-eslint/no-namespace namespace globalThis { - // eslint-disable-next-line no-var + // oxlint-disable-next-line no-var var AFFiNE: AFFiNEConfig; } } diff --git a/packages/backend/server/src/base/config/default.ts b/packages/backend/server/src/base/config/default.ts index 70186543621f0..30cf58d003ce5 100644 --- a/packages/backend/server/src/base/config/default.ts +++ b/packages/backend/server/src/base/config/default.ts @@ -30,6 +30,7 @@ function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig { 'graphql', 'sync', 'renderer', + 'doc', ]); const deploymentType = readEnv( 'DEPLOYMENT_TYPE', @@ -66,6 +67,7 @@ function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig { graphql: flavor === 'graphql' || flavor === 'allinone', sync: flavor === 'sync' || flavor === 'allinone', renderer: flavor === 'renderer' || flavor === 'allinone', + doc: flavor === 'doc' || flavor === 'allinone', }, affine, node, diff --git a/packages/backend/server/src/base/error/def.ts b/packages/backend/server/src/base/error/def.ts index 4e76e38e00c4e..155ecdb5f3dc0 100644 --- a/packages/backend/server/src/base/error/def.ts +++ b/packages/backend/server/src/base/error/def.ts @@ -101,6 +101,15 @@ export class UserFriendlyError extends Error { this.requestId = ClsServiceManager.getClsService()?.getId(); } + static fromUserFriendlyErrorJSON(body: UserFriendlyError) { + return new UserFriendlyError( + body.type as UserFriendlyErrorBaseType, + body.name.toLowerCase() as keyof typeof USER_FRIENDLY_ERRORS, + body.message, + body.data + ); + } + toJSON() { return { status: this.status, diff --git a/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts b/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts index f46a38a4d7b5b..1a18b2566a0df 100644 --- a/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts +++ b/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts @@ -50,8 +50,17 @@ test.beforeEach(async t => { test('should be able to sign and verify', t => { const data = 'hello world'; const signature = t.context.crypto.sign(data); - t.true(t.context.crypto.verify(data, signature)); - t.false(t.context.crypto.verify(data, 'fake-signature')); + t.true(t.context.crypto.verify(signature)); + t.false(t.context.crypto.verify('fake-signature')); +}); + +test('should same data should get different signature', t => { + const data = 'hello world'; + const signature = t.context.crypto.sign(data); + const signature2 = t.context.crypto.sign(data); + t.not(signature2, signature); + t.true(t.context.crypto.verify(signature)); + t.true(t.context.crypto.verify(signature2)); }); test('should be able to encrypt and decrypt', t => { diff --git a/packages/backend/server/src/base/helpers/crypto.ts b/packages/backend/server/src/base/helpers/crypto.ts index e9abc160fe861..a670e75d5d352 100644 --- a/packages/backend/server/src/base/helpers/crypto.ts +++ b/packages/backend/server/src/base/helpers/crypto.ts @@ -46,10 +46,11 @@ export class CryptoHelper { const sign = createSign('rsa-sha256'); sign.update(data, 'utf-8'); sign.end(); - return sign.sign(this.keyPair.privateKey, 'base64'); + return `${data},${sign.sign(this.keyPair.privateKey, 'base64')}`; } - verify(data: string, signature: string) { + verify(signatureWithData: string) { + const [data, signature] = signatureWithData.split(','); const verify = createVerify('rsa-sha256'); verify.update(data, 'utf-8'); verify.end(); diff --git a/packages/backend/server/src/config/affine.env.ts b/packages/backend/server/src/config/affine.env.ts index c72f5e617f57b..80fcb5543a128 100644 --- a/packages/backend/server/src/config/affine.env.ts +++ b/packages/backend/server/src/config/affine.env.ts @@ -36,6 +36,7 @@ AFFiNE.ENV_MAP = { REDIS_SERVER_PASSWORD: 'redis.password', REDIS_SERVER_DATABASE: ['redis.db', 'int'], DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'], + DOC_SERVICE_ENDPOINT: 'docService.endpoint', STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey', STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey', }; diff --git a/packages/backend/server/src/core/auth/guard.ts b/packages/backend/server/src/core/auth/guard.ts index eba48a0ffd8fa..cecbdb0f9aa59 100644 --- a/packages/backend/server/src/core/auth/guard.ts +++ b/packages/backend/server/src/core/auth/guard.ts @@ -10,8 +10,10 @@ import type { Request, Response } from 'express'; import { Socket } from 'socket.io'; import { + AccessDenied, AuthenticationRequired, Config, + CryptoHelper, getRequestResponseFromContext, parseCookies, } from '../../base'; @@ -20,12 +22,14 @@ import { AuthService } from './service'; import { Session } from './session'; const PUBLIC_ENTRYPOINT_SYMBOL = Symbol('public'); +const INTERNAL_ENTRYPOINT_SYMBOL = Symbol('internal'); @Injectable() export class AuthGuard implements CanActivate, OnModuleInit { private auth!: AuthService; constructor( + private readonly crypto: CryptoHelper, private readonly ref: ModuleRef, private readonly reflector: Reflector ) {} @@ -36,6 +40,21 @@ export class AuthGuard implements CanActivate, OnModuleInit { async canActivate(context: ExecutionContext) { const { req, res } = getRequestResponseFromContext(context); + const clazz = context.getClass(); + const handler = context.getHandler(); + // rpc request is internal + const isInternal = this.reflector.getAllAndOverride( + INTERNAL_ENTRYPOINT_SYMBOL, + [clazz, handler] + ); + if (isInternal) { + // check access token: data,signature + const accessToken = req.get('x-access-token'); + if (accessToken && this.crypto.verify(accessToken)) { + return true; + } + throw new AccessDenied('Invalid internal request'); + } const userSession = await this.signIn(req, res); if (res && userSession && userSession.expiresAt) { @@ -45,7 +64,7 @@ export class AuthGuard implements CanActivate, OnModuleInit { // api is public const isPublic = this.reflector.getAllAndOverride( PUBLIC_ENTRYPOINT_SYMBOL, - [context.getClass(), context.getHandler()] + [clazz, handler] ); if (isPublic) { @@ -85,6 +104,11 @@ export class AuthGuard implements CanActivate, OnModuleInit { */ export const Public = () => SetMetadata(PUBLIC_ENTRYPOINT_SYMBOL, true); +/** + * Mark rpc api to be internal accessible + */ +export const Internal = () => SetMetadata(INTERNAL_ENTRYPOINT_SYMBOL, true); + export const AuthWebsocketOptionsProvider: FactoryProvider = { provide: WEBSOCKET_OPTIONS, useFactory: (config: Config, guard: AuthGuard) => { diff --git a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts new file mode 100644 index 0000000000000..d6f4e2b01a372 --- /dev/null +++ b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts @@ -0,0 +1,122 @@ +import { randomUUID } from 'node:crypto'; + +import { User, Workspace } from '@prisma/client'; +import ava, { TestFn } from 'ava'; +import request from 'supertest'; + +import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; +import { AppModule } from '../../../app.module'; +import { CryptoHelper } from '../../../base'; +import { ConfigModule } from '../../../base/config'; +import { Models } from '../../../models'; + +const test = ava as TestFn<{ + models: Models; + app: TestingApp; + crypto: CryptoHelper; +}>; + +test.before(async t => { + const { app } = await createTestingApp({ + imports: [ConfigModule.forRoot(), AppModule], + }); + + t.context.models = app.get(Models); + t.context.crypto = app.get(CryptoHelper); + t.context.app = app; +}); + +let user: User; +let workspace: Workspace; + +test.beforeEach(async t => { + await t.context.app.initTestingDB(); + user = await t.context.models.user.create({ + email: 'test@affine.pro', + }); + workspace = await t.context.models.workspace.create(user.id); +}); + +test.after.always(async t => { + await t.context.app.close(); +}); + +test('should forbid access to rpc api without access token', async t => { + const { app } = t.context; + + await request(app.getHttpServer()) + .get('/rpc/workspaces/123/docs/123') + .expect({ + status: 403, + code: 'Forbidden', + type: 'NO_PERMISSION', + name: 'ACCESS_DENIED', + message: 'Invalid internal request', + }) + .expect(403); + t.pass(); +}); + +test('should forbid access to rpc api with invalid access token', async t => { + const { app } = t.context; + + await request(app.getHttpServer()) + .get('/rpc/workspaces/123/docs/123') + .set('x-access-token', 'invalid,wrong-signature') + .expect({ + status: 403, + code: 'Forbidden', + type: 'NO_PERMISSION', + name: 'ACCESS_DENIED', + message: 'Invalid internal request', + }) + .expect(403); + t.pass(); +}); + +test('should 404 when doc not found', async t => { + const { app } = t.context; + + const workspaceId = '123'; + const docId = '123'; + await request(app.getHttpServer()) + .get(`/rpc/workspaces/${workspaceId}/docs/${docId}`) + .set('x-access-token', t.context.crypto.sign(docId)) + .expect({ + status: 404, + code: 'Not Found', + type: 'RESOURCE_NOT_FOUND', + name: 'NOT_FOUND', + message: 'Doc not found', + }) + .expect(404); + t.pass(); +}); + +test('should return doc when found', async t => { + const { app } = t.context; + + const docId = randomUUID(); + const timestamp = Date.now(); + await t.context.models.doc.createUpdates([ + { + spaceId: workspace.id, + docId, + blob: Buffer.from('blob1 data'), + timestamp, + editorId: user.id, + }, + ]); + + const res = await request(app.getHttpServer()) + .get(`/rpc/workspaces/${workspace.id}/docs/${docId}`) + .set('x-access-token', t.context.crypto.sign(docId)) + .set('x-rpc-trace-id', 'test-trace-id') + .expect(200) + .expect('x-request-id', 'test-trace-id') + .expect('Content-Type', 'application/octet-stream'); + const bin = res.body as Buffer; + t.is(bin.toString(), 'blob1 data'); + t.is(res.headers['x-doc-timestamp'], timestamp.toString()); + t.is(res.headers['x-doc-editor-id'], user.id); +}); diff --git a/packages/backend/server/src/core/doc-service/__tests__/service-from-database.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/service-from-database.spec.ts new file mode 100644 index 0000000000000..8d7480527d554 --- /dev/null +++ b/packages/backend/server/src/core/doc-service/__tests__/service-from-database.spec.ts @@ -0,0 +1,72 @@ +import { randomUUID } from 'node:crypto'; + +import { User, Workspace } from '@prisma/client'; +import ava, { TestFn } from 'ava'; + +import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; +import { AppModule } from '../../../app.module'; +import { ConfigModule } from '../../../base/config'; +import { Models } from '../../../models'; +import { DocService } from '..'; +import { DatabaseDocService } from '../service'; + +const test = ava as TestFn<{ + models: Models; + app: TestingApp; + docService: DocService; +}>; + +test.before(async t => { + const { app } = await createTestingApp({ + imports: [ConfigModule.forRoot(), AppModule], + }); + + t.context.models = app.get(Models); + t.context.docService = app.get(DocService); + t.context.app = app; +}); + +let user: User; +let workspace: Workspace; + +test.beforeEach(async t => { + await t.context.app.initTestingDB(); + user = await t.context.models.user.create({ + email: 'test@affine.pro', + }); + workspace = await t.context.models.workspace.create(user.id); +}); + +test.after.always(async t => { + await t.context.app.close(); +}); + +test('should return null when doc not found', async t => { + const { docService } = t.context; + const docId = randomUUID(); + const doc = await docService.getDoc(workspace.id, docId); + t.is(doc, null); +}); + +test('should return doc when found', async t => { + const { docService } = t.context; + t.true(docService instanceof DatabaseDocService); + + const docId = randomUUID(); + const timestamp = Date.now(); + await t.context.models.doc.createUpdates([ + { + spaceId: workspace.id, + docId, + blob: Buffer.from('blob1 data'), + timestamp, + editorId: user.id, + }, + ]); + + const doc = await docService.getDoc(workspace.id, docId); + t.truthy(doc); + t.is(doc!.bin.toString(), 'blob1 data'); + t.is(doc!.timestamp, timestamp); + t.is(doc!.editor, user.id); +}); diff --git a/packages/backend/server/src/core/doc-service/__tests__/service-from-rpc.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/service-from-rpc.spec.ts new file mode 100644 index 0000000000000..43dfe18464205 --- /dev/null +++ b/packages/backend/server/src/core/doc-service/__tests__/service-from-rpc.spec.ts @@ -0,0 +1,108 @@ +import { randomUUID } from 'node:crypto'; + +import { User, Workspace } from '@prisma/client'; +import ava, { TestFn } from 'ava'; + +import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; +import { AppModule } from '../../../app.module'; +import { Config } from '../../../base'; +import { ConfigModule } from '../../../base/config'; +import { Models } from '../../../models'; +import { DocService } from '..'; +import { RpcDocService } from '../service'; + +const test = ava as TestFn<{ + models: Models; + app: TestingApp; + docService: DocService; + config: Config; +}>; + +test.before(async t => { + const { app } = await createTestingApp({ + imports: [ + ConfigModule.forRoot({ + flavor: { + doc: false, + }, + docService: { + endpoint: '', + }, + }), + AppModule, + ], + }); + + t.context.models = app.get(Models); + t.context.docService = app.get(DocService); + t.context.config = app.get(Config); + t.context.app = app; +}); + +let user: User; +let workspace: Workspace; + +test.beforeEach(async t => { + t.context.config.docService.endpoint = t.context.app.getHttpServerUrl(); + await t.context.app.initTestingDB(); + user = await t.context.models.user.create({ + email: 'test@affine.pro', + }); + workspace = await t.context.models.workspace.create(user.id); +}); + +test.after.always(async t => { + await t.context.app.close(); +}); + +test('should return null when doc not found', async t => { + const { docService } = t.context; + const docId = randomUUID(); + const doc = await docService.getDoc(workspace.id, docId); + t.is(doc, null); +}); + +test('should fallback to database doc service when endpoint network error', async t => { + const { docService } = t.context; + t.context.config.docService.endpoint = 'http://localhost:13010'; + const docId = randomUUID(); + const timestamp = Date.now(); + await t.context.models.doc.createUpdates([ + { + spaceId: workspace.id, + docId, + blob: Buffer.from('blob1 data'), + timestamp, + editorId: user.id, + }, + ]); + + const doc = await docService.getDoc(workspace.id, docId); + t.truthy(doc); + t.is(doc!.bin.toString(), 'blob1 data'); + t.is(doc!.timestamp, timestamp); + t.is(doc!.editor, user.id); +}); + +test('should return doc when found', async t => { + const { docService } = t.context; + t.true(docService instanceof RpcDocService); + + const docId = randomUUID(); + const timestamp = Date.now(); + await t.context.models.doc.createUpdates([ + { + spaceId: workspace.id, + docId, + blob: Buffer.from('blob1 data'), + timestamp, + editorId: user.id, + }, + ]); + + const doc = await docService.getDoc(workspace.id, docId); + t.truthy(doc); + t.is(doc!.bin.toString(), 'blob1 data'); + t.is(doc!.timestamp, timestamp); + t.is(doc!.editor, user.id); +}); diff --git a/packages/backend/server/src/core/doc-service/config.ts b/packages/backend/server/src/core/doc-service/config.ts new file mode 100644 index 0000000000000..64dcf3834553a --- /dev/null +++ b/packages/backend/server/src/core/doc-service/config.ts @@ -0,0 +1,19 @@ +import { defineStartupConfig, ModuleConfig } from '../../base/config'; + +interface DocServiceStartupConfigurations { + /** + * The endpoint of the doc service. + * Example: http://doc-service:3020 + */ + endpoint: string; +} + +declare module '../../base/config' { + interface AppConfig { + docService: ModuleConfig; + } +} + +defineStartupConfig('docService', { + endpoint: '', +}); diff --git a/packages/backend/server/src/core/doc-service/controller.ts b/packages/backend/server/src/core/doc-service/controller.ts new file mode 100644 index 0000000000000..15a2ded0f9b7d --- /dev/null +++ b/packages/backend/server/src/core/doc-service/controller.ts @@ -0,0 +1,30 @@ +import { Controller, Get, Param, Res } from '@nestjs/common'; +import type { Response } from 'express'; + +import { NotFound, SkipThrottle } from '../../base'; +import { Internal } from '../auth'; +import { PgWorkspaceDocStorageAdapter } from '../doc'; + +@Controller('/rpc') +export class DocRpcController { + constructor(private readonly workspace: PgWorkspaceDocStorageAdapter) {} + + @SkipThrottle() + @Internal() + @Get('/workspaces/:workspaceId/docs/:docId') + async render( + @Param('workspaceId') workspaceId: string, + @Param('docId') docId: string, + @Res() res: Response + ) { + const doc = await this.workspace.getDoc(workspaceId, docId); + if (!doc) { + throw new NotFound('Doc not found'); + } + res.setHeader('x-doc-timestamp', doc.timestamp.toString()); + if (doc.editor) { + res.setHeader('x-doc-editor-id', doc.editor); + } + res.send(doc.bin); + } +} diff --git a/packages/backend/server/src/core/doc-service/index.ts b/packages/backend/server/src/core/doc-service/index.ts new file mode 100644 index 0000000000000..8f490eb5e1b16 --- /dev/null +++ b/packages/backend/server/src/core/doc-service/index.ts @@ -0,0 +1,15 @@ +import { Module } from '@nestjs/common'; + +import { DocStorageModule } from '../doc'; +import { DocRpcController } from './controller'; +import { DocService, DocServiceProvider } from './service'; + +@Module({ + imports: [DocStorageModule], + providers: [DocServiceProvider], + controllers: [DocRpcController], + exports: [DocService], +}) +export class DocServiceModule {} + +export { DocService }; diff --git a/packages/backend/server/src/core/doc-service/service.ts b/packages/backend/server/src/core/doc-service/service.ts new file mode 100644 index 0000000000000..fcd0288cd8c1c --- /dev/null +++ b/packages/backend/server/src/core/doc-service/service.ts @@ -0,0 +1,90 @@ +import { FactoryProvider, Injectable, Logger } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; +import { ClsService } from 'nestjs-cls'; + +import { Config, CryptoHelper, UserFriendlyError } from '../../base'; +import { PgWorkspaceDocStorageAdapter } from '../doc'; +import { type DocRecord } from '../doc/storage'; + +export abstract class DocService { + abstract getDoc( + workspaceId: string, + docId: string + ): Promise; +} + +@Injectable() +export class DatabaseDocService extends DocService { + constructor(protected readonly workspace: PgWorkspaceDocStorageAdapter) { + super(); + } + + async getDoc(workspaceId: string, docId: string): Promise { + return await this.workspace.getDoc(workspaceId, docId); + } +} + +@Injectable() +export class RpcDocService extends DatabaseDocService { + private readonly logger = new Logger(DocService.name); + + constructor( + private readonly config: Config, + private readonly crypto: CryptoHelper, + private readonly cls: ClsService, + protected override readonly workspace: PgWorkspaceDocStorageAdapter + ) { + super(workspace); + } + + override async getDoc( + workspaceId: string, + docId: string + ): Promise { + const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}`; + try { + const res = await fetch(url, { + headers: { + 'x-access-token': this.crypto.sign(docId), + 'x-rpc-trace-id': this.cls.getId(), + }, + }); + if (!res.ok) { + if (res.status === 404) { + return null; + } + const body = (await res.json()) as UserFriendlyError; + throw UserFriendlyError.fromUserFriendlyErrorJSON(body); + } + const timestamp = res.headers.get('x-doc-timestamp') as string; + const editor = res.headers.get('x-doc-editor-id') as string; + const bin = await res.arrayBuffer(); + return { + spaceId: workspaceId, + docId, + bin: Buffer.from(bin), + timestamp: parseInt(timestamp), + editor, + }; + } catch (err) { + // network error or user friendly error + this.logger.error( + `Failed to fetch doc ${url}, error: ${err}`, + (err as Error).stack + ); + // fallback to database doc service + return await super.getDoc(workspaceId, docId); + } + } +} + +export const DocServiceProvider: FactoryProvider = { + provide: DocService, + useFactory: (config: Config, ref: ModuleRef) => { + if (config.flavor.doc) { + return ref.create(DatabaseDocService); + } + return ref.create(RpcDocService); + }, + inject: [Config, ModuleRef], +}; diff --git a/packages/backend/server/src/core/doc/adapters/workspace.ts b/packages/backend/server/src/core/doc/adapters/workspace.ts index 4b0d14f292309..7154c37f621f2 100644 --- a/packages/backend/server/src/core/doc/adapters/workspace.ts +++ b/packages/backend/server/src/core/doc/adapters/workspace.ts @@ -41,7 +41,9 @@ declare global { } @Injectable() export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { - private readonly logger = new Logger(PgWorkspaceDocStorageAdapter.name); + protected override readonly logger = new Logger( + PgWorkspaceDocStorageAdapter.name + ); constructor( private readonly models: Models, diff --git a/packages/backend/server/src/core/doc/storage/doc.ts b/packages/backend/server/src/core/doc/storage/doc.ts index 5ce1b7795c2fe..5ca396b14c83a 100644 --- a/packages/backend/server/src/core/doc/storage/doc.ts +++ b/packages/backend/server/src/core/doc/storage/doc.ts @@ -1,3 +1,4 @@ +import { Logger } from '@nestjs/common'; import { applyUpdate, diffUpdate, @@ -49,6 +50,7 @@ export interface DocStorageOptions { export abstract class DocStorageAdapter extends Connection { private readonly locker = new SingletonLocker(); + protected readonly logger = new Logger(DocStorageAdapter.name); constructor( protected readonly options: DocStorageOptions = { @@ -76,6 +78,9 @@ export abstract class DocStorageAdapter extends Connection { const updates = await this.getDocUpdates(spaceId, docId); if (updates.length) { + this.logger.log( + `Squashing updates, spaceId: ${spaceId}, docId: ${docId}, updates: ${updates.length}` + ); const { timestamp, bin, editor } = await this.squash( snapshot ? [snapshot, ...updates] : updates ); @@ -96,7 +101,12 @@ export abstract class DocStorageAdapter extends Connection { } // always mark updates as merged unless throws - await this.markUpdatesMerged(spaceId, docId, updates); + const count = await this.markUpdatesMerged(spaceId, docId, updates); + if (count > 0) { + this.logger.log( + `Marked ${count} updates as merged, spaceId: ${spaceId}, docId: ${docId}` + ); + } return newSnapshot; } diff --git a/packages/backend/server/src/plugins/license/service.ts b/packages/backend/server/src/plugins/license/service.ts index 59ec1a2f2a5b8..0f40bccb4ac76 100644 --- a/packages/backend/server/src/plugins/license/service.ts +++ b/packages/backend/server/src/plugins/license/service.ts @@ -345,12 +345,7 @@ export class LicenseService implements OnModuleInit { if (!res.ok) { const body = (await res.json()) as UserFriendlyError; - throw new UserFriendlyError( - body.type as any, - body.name.toLowerCase() as any, - body.message, - body.data - ); + throw UserFriendlyError.fromUserFriendlyErrorJSON(body); } const data = (await res.json()) as T;