DOP-5197

extensions/offline-snooty/package.json

@@ -16,6 +16,7 @@
     "@trpc/react-query": "^11.0.0-rc.477",
     "@trpc/server": "^11.0.0-rc.477",
     "happy-dom": "^15.10.1",
+    "mongodb": "^6.12.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "tar": "^7.4.3",

extensions/offline-snooty/src/convertGatsbyToHtml/fileHandler.ts

@@ -77,7 +77,6 @@ export const handleHtmlFile = async (
   filepath: string,
   relativePath: string,
 ) => {
-  console.log('handlehtmlfile ', filepath);
   // update the DOM. change paths for links and images
   // first open the file. as a DOM string.
   const html = (await fsPromises.readFile(filepath)).toString();
@@ -91,8 +90,6 @@ export const handleHtmlFile = async (
   updateToRelativePaths([...links, ...images], relativePath ?? './');
   removeScripts(document);
 
-  console.log('writing file html ', filepath);
-
   await fsPromises.writeFile(filepath, document.documentElement.innerHTML);
 
   await window.happyDOM.close();

extensions/offline-snooty/src/convertGatsbyToHtml/index.ts

@@ -57,7 +57,9 @@ async function scanFileTree(directoryPath: string) {
     const stat = await fsPromises.stat(filename);
     const extName = filename.split('.').pop() ?? '';
 
-    if (stat.isDirectory() || IMAGE_EXT.has(extName)) {
+    if (filename.endsWith('_gatsby') || filename.endsWith('~partytown')) {
+      fsPromises.rm(filename, { recursive: true, force: true });
+    } else if (stat.isDirectory() || IMAGE_EXT.has(extName)) {
       continue;
     }
     if (extName.endsWith('html')) {
@@ -73,8 +75,6 @@ async function scanFileTree(directoryPath: string) {
     } else {
       // delete the file
       await fsPromises.rm(filename);
-      console.log('removing file ', filename);
-
       log.removedFiles.push(filename);
     }
   }

extensions/offline-snooty/src/createSnootyCopy/index.ts

@@ -1,11 +1,12 @@
 import type { NetlifyPluginUtils } from '@netlify/build';
+import { promises } from 'node:fs';
 
 export const createSnootyCopy = async (
   run: NetlifyPluginUtils['run'],
   targetPath: string,
 ) => {
   await run.command(
-    `rsync -av ${process.cwd()}/snooty ${targetPath} --exclude public --exclude node_modules`,
+    `rsync -a ${process.cwd()}/snooty ${targetPath} --exclude public --exclude node_modules`,
   );
 
   const offlineSnootyPath = `${targetPath}/snooty`;
@@ -18,6 +19,13 @@ export const createSnootyCopy = async (
     cwd: offlineSnootyPath,
   });
 
+  await promises.appendFile(
+    `${offlineSnootyPath}/.env.production`,
+    '\nOFFLINE_DOCS=true\n',
+  );
+
+  await run.command('cat ./.env.production', { cwd: offlineSnootyPath });
+
   await run.command('npm run build:no-prefix', {
     cwd: offlineSnootyPath,
   });

extensions/offline-snooty/src/index.ts

@@ -1,48 +1,75 @@
 // Documentation: https://sdk.netlify.com
-import { NetlifyExtension } from '@netlify/sdk';
+import type {
+  DocsetsDocument,
+  ReposBranchesDocument,
+  BranchEntry,
+  EnvironmentConfig,
+} from 'util/databaseConnection/types';
+import { Extension, envVarToBool } from 'util/extension';
 import { convertGatsbyToHtml } from './convertGatsbyToHtml';
 import { createSnootyCopy } from './createSnootyCopy';
+import { updateReposBranches } from './updateReposBranches';
 import { destroyClient, uploadToS3 } from './uploadToS3';
-import {
-  type BranchEntry,
-  type DocsetsDocument,
-  type ReposBranchesDocument,
-  readEnvConfigs,
-} from './uploadToS3/utils';
+import { readEnvConfigs } from './uploadToS3/utils';
 
-const extension = new NetlifyExtension();
+const extension = new Extension({
+  isEnabled: envVarToBool(process.env.OFFLINE_SNOOTY_ENABLED),
+});
 const NEW_SNOOTY_PATH = `${process.cwd()}/snooty-offline`;
 export const PUBLIC_OUTPUT_PATH = `${NEW_SNOOTY_PATH}/snooty/public`;
+const EXTENSION_NAME = 'offline-snooty';
+const ENVS_TO_RUN = ['dotcomprd', 'dotcomstg'];
 
 // run this extension after the build and deploy are successful
 extension.addBuildEventHandler(
   'onSuccess',
-  async ({ netlifyConfig, utils: { run } }) => {
-    // If the build event handler is not enabled, return early
-    if (!process.env.OFFLINE_SNOOTY_ENABLED) {
-      return;
-    }
-
+  async ({ netlifyConfig, dbEnvVars, utils: { run } }) => {
     const environment = netlifyConfig.build.environment as Record<
       string,
       string | DocsetsDocument | ReposBranchesDocument | BranchEntry
     >;
-    const { bucketName, fileName } = readEnvConfigs({
-      env: (environment.ENV as string) ?? '',
-      docsetEntry: (environment.DOCSET_ENTRY as DocsetsDocument) ?? {},
+    // NOTE: prd and dotcomprd both read from pool.repos_branches
+    // would be an improvement to separate prd and dotcomprd repos_branches
+    // skip this step if step is `prd`
+    // could only test in dotcomstg :(
+    // if (!ENVS_TO_RUN.includes(environment.ENV as string)) {
+    //   console.log('skipping repos branches update');
+    //   return;
+    // }
+    const { bucketName, fileName, baseUrl } = readEnvConfigs({
+      env: (environment.ENV as keyof EnvironmentConfig) ?? '',
       repoEntry: (environment.REPO_ENTRY as ReposBranchesDocument) ?? {},
       branchEntry: (environment.BRANCH_ENTRY as BranchEntry) ?? {},
+      docsetEntry: (environment.DOCSET_ENTRY as DocsetsDocument) ?? {},
     });
 
     try {
       console.log('... creating snooty copy');
       await createSnootyCopy(run, NEW_SNOOTY_PATH);
       console.log('... converting gatsby to html');
       await convertGatsbyToHtml(PUBLIC_OUTPUT_PATH, fileName);
-      console.log('... uploading to AWS S3 ', bucketName, fileName);
+      console.log(
+        '... uploading to AWS S3 ',
+        bucketName,
+        'docs/offline',
+        fileName,
+      );
       await uploadToS3(`${process.cwd()}/${fileName}`, bucketName, fileName);
       console.log('... uploaded to AWS S3');
-      // TODO: update atlas collection repos_branches to signal offline availability
+      await updateReposBranches(
+        {
+          repoEntry: environment.REPO_ENTRY as ReposBranchesDocument,
+          branchEntry: environment.BRANCH_ENTRY as BranchEntry,
+          collectionName: dbEnvVars.REPOS_BRANCHES_COLLECTION,
+        },
+        {
+          clusterZeroURI: dbEnvVars.ATLAS_CLUSTER0_URI,
+          databaseName: netlifyConfig?.build?.environment.POOL_DB_NAME ?? '',
+          appName: EXTENSION_NAME,
+        },
+        baseUrl,
+        fileName,
+      );
     } catch (e) {
       console.error(e);
       throw e;

extensions/offline-snooty/src/updateReposBranches/index.ts

@@ -0,0 +1,52 @@
+import { ObjectId, type UpdateResult } from 'mongodb';
+import { join } from 'node:path';
+import type {
+  BranchEntry,
+  ReposBranchesDocument,
+} from 'util/databaseConnection/types';
+import {
+  getClusterZeroDb,
+  type clusterZeroParams,
+} from 'util/databaseConnection/clusterZeroConnector';
+import type { CollectionName } from 'util/assertDbEnvVars';
+
+type updateParams = {
+  branchEntry: BranchEntry;
+  repoEntry: ReposBranchesDocument;
+  collectionName?: CollectionName;
+};
+
+function getUrl(baseUrl: string, fileName: string) {
+  return join(baseUrl, 'docs', 'offline', fileName);
+}
+
+export const updateReposBranches = async (
+  { repoEntry, branchEntry, collectionName }: updateParams,
+  connectionParams: clusterZeroParams,
+  baseUrl: string,
+  fileName: string,
+): Promise<UpdateResult<ReposBranchesDocument>> => {
+  const dbSession = await getClusterZeroDb(connectionParams);
+  const reposBranchesCollection = dbSession.collection<ReposBranchesDocument>(
+    collectionName ?? 'repos_branches',
+  );
+
+  const updateFilter = {
+    _id: new ObjectId(repoEntry._id),
+    'branches.id': new ObjectId(branchEntry.id),
+  };
+
+  const updateParams = {
+    $set: { 'branches.$.offlineUrl': getUrl(baseUrl, fileName) },
+  };
+
+  console.log(
+    `Updating repos branches collection for collection ${collectionName} in db ${connectionParams.databaseName} for repo id ${repoEntry._id} for branch ${branchEntry.id} ${branchEntry.gitBranchName}`,
+  );
+
+  const updateRes = await reposBranchesCollection.updateOne(
+    updateFilter,
+    updateParams,
+  );
+  return updateRes;
+};

extensions/offline-snooty/src/uploadToS3/utils.ts

@@ -1,52 +1,37 @@
-// TODO: consolidate with populate-metadata extension
-type EnvironmentConfig = {
-  dev?: string;
-  stg: string;
-  dotcomstg: string;
-  prd: string;
-  dotcomprd: string;
-};
-export interface DocsetsDocument {
-  bucket: EnvironmentConfig;
-  project: string;
-  url: EnvironmentConfig;
-  prefix: EnvironmentConfig;
-}
+import type {
+  ReposBranchesDocument,
+  BranchEntry,
+  EnvironmentConfig,
+  DocsetsDocument,
+} from 'util/databaseConnection/types';
 
-export interface BranchEntry {
-  name?: string;
-  gitBranchName: string;
-  urlSlug: string;
-  isStableBranch: boolean;
-  active: boolean;
-}
-
-export interface ReposBranchesDocument {
-  repoName: string;
-  project: string;
-  search?: {
-    categoryTitle: string;
-    categoryName?: string;
-  };
-  branches?: Array<BranchEntry>;
-  prodDeployable: boolean;
-  internalOnly: boolean;
+/**
+ * Returns buckets that are preconfigured to public URL
+ * Some projects are stored in different buckets,
+ * but offline versions are stored in general usage bucket
+ * to route to same URL
+ */
+function getBucketName(env: keyof EnvironmentConfig) {
+  return `docs-mongodb-org-${env}`;
 }
 
 export function readEnvConfigs({
   env,
-  docsetEntry,
   repoEntry,
   branchEntry,
+  docsetEntry,
 }: {
-  env: string;
-  docsetEntry: DocsetsDocument;
+  env: keyof EnvironmentConfig;
   repoEntry: ReposBranchesDocument;
   branchEntry: BranchEntry;
+  docsetEntry: DocsetsDocument;
 }) {
-  const docset: DocsetsDocument = docsetEntry;
-  const bucketName = docset.bucket[env as keyof EnvironmentConfig] ?? '';
+  const bucketName = getBucketName(env);
   const project: string = repoEntry?.project ?? '';
   const version = branchEntry?.gitBranchName ?? '';
-  return { bucketName, fileName: `${project}-${version}.tar.gz` };
+  return {
+    bucketName,
+    fileName: `${project}-${version}.tar.gz`,
+    baseUrl: docsetEntry?.url[env] ?? '',
+  };
 }

extensions/populate-metadata/src/getProperties.ts

@@ -65,7 +65,7 @@ const getRepoEntry = async ({
   };
   const projection = {
     projection: {
-      _id: 0,
+      _id: 1,
       repoName: 1,
       branches: { $elemMatch: { gitBranchName: branchName.toLowerCase() } },
       project: 1,

libs/util/src/databaseConnection/clusterZeroConnector.ts

@@ -3,18 +3,23 @@ import { teardown, dbClient } from './clusterConnector';
 
 let clusterZeroClient: mongodb.MongoClient;
 
+export type clusterZeroParams = {
+  clusterZeroURI: string;
+  databaseName: string;
+  appName?: string;
+};
+
 export const getClusterZeroDb = async ({
   clusterZeroURI,
   databaseName,
   appName,
-}: {
-  clusterZeroURI: string;
-  databaseName: string;
-  appName: string;
-}): Promise<mongodb.Db> => {
+}: clusterZeroParams): Promise<mongodb.Db> => {
   if (!clusterZeroClient) {
     console.info('Creating new instance of Cluster Zero client');
-    clusterZeroClient = await dbClient({ uri: clusterZeroURI, appName });
+    clusterZeroClient = await dbClient({
+      uri: clusterZeroURI,
+      appName: appName ?? '',
+    });
   }
   return clusterZeroClient.db(databaseName);
 };

libs/util/src/databaseConnection/types.ts

@@ -32,6 +32,7 @@ export type SearchClusterConnectionInfo = {
 };
 
 export type BranchEntry = {
+  id?: ObjectId;
   name?: string;
   gitBranchName: string;
   active: boolean;
@@ -42,6 +43,7 @@ export type BranchEntry = {
 };
 
 export type ReposBranchesDocument = {
+  _id?: ObjectId;
   repoName: string;
   project: string;
   search?: {