feat: Add heartbeat endpoint & tests (#881)

* Restructure jobs router + Add heartbeat * Change heartbeat to use DD * Update src/jobs/index.ts Co-authored-by: Ian Woodard <[email protected]> * Update nits * Update alert type & nits * Add sentry crons? * Fix renaming --------- Co-authored-by: Ian Woodard <[email protected]>
getsentry · Oct 23, 2024 · cf2e2fb · cf2e2fb
1 parent 5fadb11
commit cf2e2fb
Show file tree

Hide file tree

Showing 5 changed files with 187 additions and 61 deletions.
diff --git a/src/jobs/README.md b/src/jobs/README.md
@@ -2,36 +2,21 @@
 
 Files in this subdirectory contain code for webhooks which trigger cron jobs. They are ran by Cloud Scheduler.
 
-One of the available cron jobs is `stale-triage-notifier` (defined in `slackNotificaitons.ts`) with a payload in the following shape:
-
-```ts
-type PubSubPayload = {
-  name: string;
-  slo?: number;
-  repos?: string[];
-};
-```
-
-This payload will be sent regularly using the [Cloud Scheduler][cloud_scheduler]
-to notify product owners about their issues pending triage over [our SLO][process_doc].
-
-[cloud_scheduler]: https://cloud.google.com/scheduler/docs/tut-pub-sub#create_a_job
-[process_doc]: https://www.notion.so/sentry/Engaging-Customers-177c77ac473e41eabe9ca7b4bf537537#9d7b15dec9c345618b9195fb5c785e53
-
 ## List of all Cron Jobs
 
-| Job Name                         | Route                            | Files                             |
-| -------------------------------- | -------------------------------- | --------------------------------- |
-| `stale-triage-notifier`          | `/jobs/stale-triage-notifier`    | `slackNotifications.ts`           |
-| `stale-bot`                      | `/jobs/stale-bot`                | `stalebot.ts`                     |
-| `slack-scores`                   | `/jobs/slack-scores`             | `slackScores.ts`                  |
-| `gocd-paused-pipeline-bot`       | `/jobs/gocd-paused-pipeline-bot` | `gocdPausedPipelineBot.ts`        |
+| Job Name                         | Route                            | Folders                         |
+| -------------------------------- | -------------------------------- | ------------------------------- |
+| `stale-triage-notifier`          | `/jobs/stale-triage-notifier`    | `/staleTriageNotifier`          |
+| `stale-bot`                      | `/jobs/stale-bot`                | `/staleBot`                     |
+| `slack-scores`                   | `/jobs/slack-scores`             | `/slackScores`                  |
+| `gocd-paused-pipeline-bot`       | `/jobs/gocd-paused-pipeline-bot` | `/gocdPausedPipeline`           |
+| `heartbeat`                      | `/jobs/heartbeat`                | `/heartbeat`                    |
 
 ## Development
 
 To add a new cron job:
 
-* Create a unique file in this subdirectory
+* Create a new file and corresponding test file in this subdirectory
 * In this file, export a function:
 
 ```ts
@@ -41,14 +26,27 @@ export async function cronJobName(
 ){}
 ```
 
-(`org` and `now` are are used for some reason by all of the other cron jobs, but you can just rename them to `_org` and `_now`)
-This function should run logic a single time when it is called.
+or
+
+```ts
+export async function cronJobName(){}
+```
+
+Depending on if your job is a Github-related job, or a generic cron job. This function should run logic a single time when it is called.
 
 * In `index.ts`, import the function and add the following code to the `routeJobs` function:
 
 ```ts
 server.post('/cron-job-path', (request, reply) =>
-  handleJobRoute(cronJobName, request, reply)
+  handleGithubJobs(cronJobName, request, reply)
+);
+```
+
+or
+
+```ts
+server.post('/cron-job-path', (request, reply) =>
+  handleCronJobs(cronJobName, request, reply)
 );
 ```
 

diff --git a/src/jobs/heartbeat.test.ts b/src/jobs/heartbeat.test.ts
@@ -0,0 +1,37 @@
+import moment from 'moment-timezone';
+
+import { DATADOG_API_INSTANCE } from '@/config';
+
+import { callDatadog } from './heartbeat';
+
+describe('test uptime heartbeat', function () {
+  let datadogApiInstanceSpy;
+  beforeEach(() => {
+    datadogApiInstanceSpy = jest
+      .spyOn(DATADOG_API_INSTANCE, 'createEvent')
+      .mockImplementation(jest.fn());
+  });
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+  it('should send a message to slack', async () => {
+    const timestamp = moment().unix();
+    await callDatadog(timestamp);
+    expect(datadogApiInstanceSpy).toHaveBeenCalledTimes(1);
+    const message = datadogApiInstanceSpy.mock.calls[0][0];
+    expect(message).toEqual({
+      body: {
+        title: 'Infra Hub Update',
+        text: 'Infra Hub is up',
+        dateHappened: timestamp,
+        alertType: 'info',
+        tags: [
+          `source_tool:infra-hub`,
+          `source:infra-hub`,
+          `source_category:infra-tools`,
+          `sentry_user:infra-hub`,
+        ],
+      },
+    });
+  });
+});
diff --git a/src/jobs/heartbeat.ts b/src/jobs/heartbeat.ts
@@ -0,0 +1,55 @@
+import { v1 } from '@datadog/datadog-api-client';
+import * as Sentry from '@sentry/node';
+import moment from 'moment-timezone';
+
+import { DATADOG_API_INSTANCE } from '@/config';
+
+export async function callDatadog(timestamp: number): Promise<void> {
+  const params: v1.EventCreateRequest = {
+    title: 'Infra Hub Update',
+    text: 'Infra Hub is up',
+    alertType: 'info',
+    dateHappened: timestamp,
+    tags: [
+      'source_tool:infra-hub',
+      'source:infra-hub',
+      'source_category:infra-tools',
+      'sentry_user:infra-hub',
+    ],
+  };
+  await DATADOG_API_INSTANCE.createEvent({ body: params });
+}
+
+export async function heartbeat() {
+  const checkInId = Sentry.captureCheckIn(
+    {
+      monitorSlug: 'infra-hub-heartbeat',
+      status: 'in_progress',
+    },
+    {
+      schedule: {
+        type: 'crontab',
+        value: '*/5 * * * *',
+      },
+      checkinMargin: 1,
+      maxRuntime: 1,
+      timezone: 'America/Los_Angeles',
+    }
+  );
+  try {
+    await callDatadog(moment().unix());
+
+    Sentry.captureCheckIn({
+      checkInId,
+      monitorSlug: 'infra-hub-heartbeat',
+      status: 'ok',
+    });
+  } catch (err) {
+    Sentry.captureException(err);
+    Sentry.captureCheckIn({
+      checkInId,
+      monitorSlug: 'infra-hub-heartbeat',
+      status: 'error',
+    });
+  }
+}
diff --git a/src/jobs/index.test.ts b/src/jobs/index.test.ts
@@ -7,18 +7,20 @@ import { OAuth2Client } from 'google-auth-library';
 
 import * as gocdPausedPipelineBot from './gocdPausedPipelineBot';
 import { triggerPausedPipelineBot } from './gocdPausedPipelineBot';
+import * as heartbeat from './heartbeat';
 import * as slackNotifications from './slackNotifications';
 import { notifyProductOwnersForUntriagedIssues } from './slackNotifications';
 import * as slackScores from './slackScores';
 import { triggerSlackScores } from './slackScores';
 import * as staleBot from './stalebot';
 import { triggerStaleBot } from './stalebot';
-import { handleJobRoute, routeJobs } from '.';
+import { handleGitHubJob, routeJobs } from '.';
 
 const mockGocdPausedPipelineBot = jest.fn();
 const mockNotifier = jest.fn();
 const mockSlackScores = jest.fn();
 const mockStaleBot = jest.fn();
+const mockHeartbeat = jest.fn();
 
 jest
   .spyOn(gocdPausedPipelineBot, 'triggerPausedPipelineBot')
@@ -30,6 +32,7 @@ jest
   .spyOn(slackScores, 'triggerSlackScores')
   .mockImplementation(mockSlackScores);
 jest.spyOn(staleBot, 'triggerStaleBot').mockImplementation(mockStaleBot);
+jest.spyOn(heartbeat, 'heartbeat').mockImplementation(mockHeartbeat);
 
 class MockReply {
   statusCode: number = 0;
@@ -62,7 +65,7 @@ describe('cron jobs testing', function () {
       },
     } as FastifyRequest<{ Body: { message: { data: string } } }>;
     const reply = new MockReply() as FastifyReply;
-    await handleJobRoute(mapOperation(operationSlug), request, reply);
+    await handleGitHubJob(mapOperation(operationSlug), request, reply);
     return reply;
   }
   let server: FastifyInstance;
@@ -135,7 +138,7 @@ describe('cron jobs testing', function () {
       headers: {},
     } as FastifyRequest<{ Body: { message: { data: string } } }>;
     const reply = new MockReply() as FastifyReply;
-    await handleJobRoute(mapOperation('stale-bot'), request, reply);
+    await handleGitHubJob(mapOperation('stale-bot'), request, reply);
     expect(reply.statusCode).toBe(400);
   });
 
@@ -153,7 +156,7 @@ describe('cron jobs testing', function () {
       },
     } as FastifyRequest<{ Body: { message: { data: string } } }>;
     const reply = new MockReply() as FastifyReply;
-    await handleJobRoute(mapOperation('stale-bot'), request, reply);
+    await handleGitHubJob(mapOperation('stale-bot'), request, reply);
     expect(reply.statusCode).toBe(400);
   });
 
@@ -227,4 +230,20 @@ describe('cron jobs testing', function () {
     expect(mockSlackScores).not.toHaveBeenCalled();
     expect(mockStaleBot).not.toHaveBeenCalled();
   });
+
+  it('POST /heartbeat should call uptime heartbeat', async () => {
+    const reply = await server.inject({
+      method: 'POST',
+      url: '/jobs/heartbeat',
+      headers: {
+        authorization: 'Bearer 1234abcd',
+      },
+    });
+    expect(reply.statusCode).toBe(204);
+    expect(mockGocdPausedPipelineBot).not.toHaveBeenCalled();
+    expect(mockNotifier).not.toHaveBeenCalled();
+    expect(mockSlackScores).not.toHaveBeenCalled();
+    expect(mockStaleBot).not.toHaveBeenCalled();
+    expect(mockHeartbeat).toHaveBeenCalled();
+  });
 });
diff --git a/src/jobs/index.ts b/src/jobs/index.ts
@@ -9,17 +9,15 @@ import { GH_ORGS } from '@/config';
 import { Fastify } from '@/types';
 
 import { triggerPausedPipelineBot } from './gocdPausedPipelineBot';
+import { heartbeat } from './heartbeat';
 import { notifyProductOwnersForUntriagedIssues } from './slackNotifications';
 import { triggerSlackScores } from './slackScores';
 import { triggerStaleBot } from './stalebot';
 
-// Error handling wrapper function
-// Additionally handles Auth from Cloud Scheduler
-export async function handleJobRoute(
-  handler,
+async function handleCronAuth(
   request: FastifyRequest,
   reply: FastifyReply
-) {
+): Promise<boolean> {
   try {
     const client = new OAuth2Client();
     // Get the Cloud Scheduler JWT in the "Authorization" header.
@@ -28,15 +26,15 @@ export async function handleJobRoute(
     if (!bearer) {
       reply.code(400);
       reply.send();
-      return;
+      return false;
     }
 
     const match = bearer.match(/Bearer (.*)/);
 
     if (!match) {
       reply.code(400);
       reply.send();
-      return;
+      return false;
     }
 
     const token = match[1];
@@ -54,7 +52,21 @@ export async function handleJobRoute(
   } catch (e) {
     reply.code(401);
     reply.send();
-    return;
+    return false;
+  }
+  return true;
+}
+
+// Error handling wrapper function for Cron Jobs involving Github
+// Additionally handles Auth from Cloud Scheduler
+export async function handleGitHubJob(
+  handler,
+  request: FastifyRequest,
+  reply: FastifyReply
+) {
+  const verified = await handleCronAuth(request, reply);
+  if (!verified) {
+    return; // Reply is already sent, so no need to re-send
   }
   const tx = Sentry.startTransaction({
     op: 'webhooks',
@@ -74,39 +86,44 @@ export async function handleJobRoute(
   tx.finish();
 }
 
-export const opts = {
-  schema: {
-    body: {
-      type: 'object',
-      required: ['message'],
-      properties: {
-        message: {
-          type: 'object',
-          required: ['data'],
-          properties: {
-            data: {
-              type: 'string',
-            },
-          },
-        },
-      },
-    },
-  },
-};
+// Error handling wrapper function for Cron Jobs
+// Additionally handles Auth from Cloud Scheduler
+export async function handleCronJob(
+  handler,
+  request: FastifyRequest,
+  reply: FastifyReply
+) {
+  const verified = await handleCronAuth(request, reply);
+  if (!verified) {
+    return; // Reply is already sent, so no need to re-send
+  }
+  const tx = Sentry.startTransaction({
+    op: 'webhooks',
+    name: 'jobs.jobsHandler',
+  });
+
+  reply.code(204);
+  reply.send(); // Respond early to not block the webhook sender
+  await handler();
+  tx.finish();
+}
 
 // Function that creates a sub fastify server for job webhooks
 export async function routeJobs(server: Fastify, _options): Promise<void> {
   server.post('/stale-triage-notifier', (request, reply) =>
-    handleJobRoute(notifyProductOwnersForUntriagedIssues, request, reply)
+    handleGitHubJob(notifyProductOwnersForUntriagedIssues, request, reply)
   );
   server.post('/stale-bot', (request, reply) =>
-    handleJobRoute(triggerStaleBot, request, reply)
+    handleGitHubJob(triggerStaleBot, request, reply)
   );
   server.post('/slack-scores', (request, reply) =>
-    handleJobRoute(triggerSlackScores, request, reply)
+    handleGitHubJob(triggerSlackScores, request, reply)
   );
   server.post('/gocd-paused-pipeline-bot', (request, reply) =>
-    handleJobRoute(triggerPausedPipelineBot, request, reply)
+    handleGitHubJob(triggerPausedPipelineBot, request, reply)
+  );
+  server.post('/heartbeat', (request, reply) =>
+    handleCronJob(heartbeat, request, reply)
   );
 
   // Default handler for invalid routes