feat(cu): allow running with embedded PouchDB or remote CoucbhDB

.gitpod.yml

@@ -5,6 +5,16 @@ tasks:
   - name: Init Repo Tooling
     init: npm i
 
+  - name: Start CU Database
+    init: docker build -t cu-couch servers/cu/.couchdb
+    command:
+      mkdir -p /workspace/cu-data
+      docker run -it \
+        -p 5984:5984 \
+        -v /workspace/cu-data:/opt/couchdb/data \
+        --env-file servers/cu/.couchdb/couchdb.conf \
+        cu-couch
+
   - name: Start MU Database
     init: docker build -t mu-postgres servers/mu/.postgres
     command:

servers/cu/.couchdb/Dockerfile

@@ -0,0 +1,4 @@
+FROM couchdb:3.3.3
+
+RUN echo '[couchdb]' > /opt/couchdb/etc/local.d/10-single-node.ini
+RUN echo 'single_node=true' >> /opt/couchdb/etc/local.d/10-single-node.ini

servers/cu/.couchdb/couchdb.conf

@@ -0,0 +1,2 @@
+COUCHDB_USER=admin
+COUCHDB_PASSWORD=password

servers/cu/.env.example

@@ -3,7 +3,8 @@ WALLET="*****"
 GATEWAY_URL="https://arweave.net"
 NODE_CONFIG_ENV="development"
 DEBUG=*
-DB_PATH="ao-cache"
+DB_MODE="embedded"
+DB_URL="ao-cache"
 DB_MAX_LISTENERS=100
 DUMP_PATH="static"
 NODE_HEAPDUMP_OPTIONS="nosignal"

servers/cu/README.md

@@ -5,8 +5,10 @@ Server.
 
 <!-- toc -->
 
+- [Getting Started](#getting-started)
 - [Usage](#usage)
 - [Environment Variables](#environment-variables)
+  - [Running With CouchDB](#running-with-couchdb)
 - [Tests](#tests)
 - [Debug Logging](#debug-logging)
 - [Heap Snapshot](#heap-snapshot)
@@ -19,6 +21,10 @@ Server.
 
 <!-- tocstop -->
 
+## Getting Started
+
+> For ease of development, this Compute Unit will use an embedded PouchDB, by default, when starting in development mode
+
 ## Usage
 
 First install dependencies using `npm i`
@@ -35,13 +41,47 @@ There are a few environment variables that you can set:
   development mode)
 - `WALLET`: the JWK Interface stringified JSON that will be used by the CU
 - `PORT`: Which port the web server should listen on (defaults to port `3005`)
-- `DB_PATH`: where on the local filesystem to place the PouchDB, which is used
-  for persistence. Defaults to `./ao-cache`
+- `DB_MODE`: the database mode to run the service in. Can be either `embedded` or `remote` (defaults to `embedded` during development)
+- `DB_URL`: The connection string to the database (when using `DB_MODE=embedded`, defaults to `ao-cache`)
 - `DB_MAX_LISTENERS`: the maximum number of event listeners for DB events.
   Defaults to `100`
 - `DUMP_PATH`: the path to send `heap` snapshots to. (See
   [Heap Snapshots](#heap-snapshot))
 
+### Running With CouchDB
+
+This Compute Unit can be ran using a remote CouchDB. Simply set set `DB_MODE=remote` and `DB_URL` to the CouchDB connection string.
+
+You will need a CouchDB database running. For convenience, a CouchDB
+`Dockefile` and configuration is included in the `.couchdb` directory that you
+can use to spin up a local Postgres instance
+
+> The `.couchdb` directory is purely a convenience for local development
+
+> If you use Gitpod, this is already done for you, as part of spinning up a new
+> workspace
+
+First, build the image by running this at the root of the `mu` module:
+
+```sh
+docker build -t cu-couchdb .couchdb
+```
+
+Then start up a container using that image. You can optionally mount a local
+directory for CouchDB to store persistent data ie. `/workspace/cu-data`
+
+```sh
+mkdir -p /workspace/cu-data
+docker run -it \
+  -p 5984:5984 \
+  -v /workspace/cu-data:/opt/couchdb/data \
+  --env-file servers/cu/.couchdb/couchdb.conf \
+  cu-couch
+```
+
+This will start a CouchDB database listening on port `5432` with credentials in
+the `./couchdb/couchdb.conf` file
+
 ## Tests
 
 You can execute unit tests by running `npm test`

servers/cu/package.json

@@ -25,6 +25,7 @@
     "lru-cache": "^10.1.0",
     "ms": "^2.1.3",
     "pouchdb": "^8.0.1",
+    "pouchdb-adapter-http": "^8.0.1",
     "pouchdb-adapter-leveldb": "^8.0.1",
     "pouchdb-find": "^8.0.1",
     "ramda": "^0.29.1",

servers/cu/src/config.js

@@ -42,7 +42,8 @@ const CONFIG_ENVS = {
     MODE,
     port: process.env.PORT || 3005,
     GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net',
-    DB_PATH: process.env.DB_PATH || 'ao-cache',
+    DB_MODE: process.env.DB_MODE || 'embedded',
+    DB_URL: process.env.DB_URL || 'ao-cache',
     DB_MAX_LISTENERS: parseInt(process.env.DB_MAX_LISTENERS || '100'),
     DUMP_PATH: process.env.DUMP_PATH || './static',
     WALLET: process.env.WALLET
@@ -51,7 +52,8 @@ const CONFIG_ENVS = {
     MODE,
     port: process.env.PORT || 3005,
     GATEWAY_URL: process.env.GATEWAY_URL,
-    DB_PATH: process.env.DB_PATH,
+    DB_MODE: process.env.DB_MODE || 'remote',
+    DB_URL: process.env.DB_URL,
     DB_MAX_LISTENERS: parseInt(process.env.DB_MAX_LISTENERS || 'throw'),
     DUMP_PATH: process.env.DUMP_PATH,
     WALLET: process.env.WALLET

servers/cu/src/domain/client/pouchdb.js

@@ -7,7 +7,8 @@ import { z } from 'zod'
 
 import PouchDb from 'pouchdb'
 import PouchDbFind from 'pouchdb-find'
-import LevelDb from 'pouchdb-adapter-leveldb'
+import PouchDbHttp from 'pouchdb-adapter-http'
+import PouchDbLevel from 'pouchdb-adapter-leveldb'
 
 import { evaluationSchema, processSchema } from '../model.js'
 
@@ -18,13 +19,25 @@ const inflateP = promisify(inflate)
  * An implementation of the db client using pouchDB
  */
 let internalPouchDb
-export function createPouchDbClient ({ maxListeners, path }) {
+export function createPouchDbClient ({ logger, maxListeners, mode, url }) {
   if (internalPouchDb) return internalPouchDb
 
-  PouchDb.plugin(LevelDb)
+  let adapter
+  if (mode === 'embedded') {
+    logger('Using embedded PouchDB')
+    PouchDb.plugin(PouchDbLevel)
+    adapter = 'leveldb'
+  } else if (mode === 'remote') {
+    logger('Using remote CouchDB')
+    PouchDb.plugin(PouchDbHttp)
+    adapter = 'http'
+  } else {
+    throw new Error(`Unsupported db mode: '${mode}'`)
+  }
+
   PouchDb.plugin(PouchDbFind)
   PouchDb.setMaxListeners(maxListeners)
-  internalPouchDb = new PouchDb(path, { adapter: 'leveldb' })
+  internalPouchDb = new PouchDb(url, { adapter })
   return internalPouchDb
 }
 

servers/cu/src/domain/index.js

@@ -33,7 +33,12 @@ export const createApis = (ctx) => {
       (processId) => locate(processId).catch(err => err)
     ))
   })
-  const pouchDb = PouchDbClient.createPouchDbClient({ maxListeners: ctx.DB_MAX_LISTENERS, path: ctx.DB_PATH })
+  const pouchDb = PouchDbClient.createPouchDbClient({
+    logger: ctx.logger,
+    mode: ctx.DB_MODE,
+    maxListeners: ctx.DB_MAX_LISTENERS,
+    url: ctx.DB_URL
+  })
 
   const sharedDeps = (logger) => ({
     loadTransactionMeta: GatewayClient.loadTransactionMetaWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, logger }),

servers/cu/src/domain/model.js

@@ -2,7 +2,8 @@ import { z } from 'zod'
 
 export const domainConfigSchema = z.object({
   GATEWAY_URL: z.string().url('GATEWAY_URL must be a a valid URL'),
-  DB_PATH: z.string().min(1, 'DB_PATH set to the location of the database on disk must be provided'),
+  DB_MODE: z.enum(['remote', 'embedded']),
+  DB_URL: z.string().min(1, 'DB_URL must be set to the database connection string'),
   DB_MAX_LISTENERS: z.number().int('DB_MAX_LISTENERS must be an integer'),
   WALLET: z.string().min(1, 'WALLET must be a Wallet JWK Inteface')
 })