Add RingCentral token refresh error logging.

Allows setting the maximum age of voicemails to fetch on the first run.

Other changes:
- Refactor `from` to `age` in function parameters
- Improve README

README.md

@@ -5,7 +5,7 @@
 ## Configuration
 
 ```shell
-# Required API/authentication variables:
+# Required API endpoint/authentication variables
 SONAR_URL=https://instance.sonar.software/api/graphql
 SONAR_TOKEN=
 RC_APP_KEY=
@@ -17,15 +17,25 @@ RC_LOGIN_PASSWORD=
 # Set to any value to enable use of RingCentral's sandbox API
 RC_SANDBOX=
 
+# The database to use
+# valid options: pg, sqlite
+# default: sqlite
 DB_ENGINE=sqlite # can be pg
-# only used when DB_ENGINE=pg
+
+# Only used when DB_ENGINE=pg
 DB_URL=
-# only used when DB_ENGINE=sqlite
+# Only used when DB_ENGINE=sqlite
+# default: voicemails.db
 DB_FILE=voicemails.db
 
 # A mapping of extension number to Sonar Ticket Group
 # Only the voicemail boxes of these extensions will be checked
 EXTENSION_TICKET_GROUPS=1:1,2:2,2:3
+
+# Upon first run, query RingCentral voicemails up to FIRST_RUN_AGE seconds old.
+# Useful when the application is restarted after not running for some time.
+# default: 86400 (1 day)
+FIRST_RUN_AGE=86400
 ```
 
 ## Deployment
@@ -45,6 +55,8 @@ services:
     environment:
       # ... see Configuration above
       DB_FILE: /data/voicemails.db
+      # so the created tickets show the correct 'Received' date & time
+      TZ: America/Creston
     volumes:
       - data:/data
 ```

src/db/migrations/20210311061107_create_audio_table.ts

@@ -0,0 +1,13 @@
+import { Knex } from "knex";
+
+export async function up(knex: Knex) {
+  await knex.schema.createTable("recordings", (table) => {
+    table.bigInteger("messageId").primary().references("voicemails.messageId");
+    table.string("mimeType", 32);
+    table.binary("audio");
+  });
+}
+
+export async function down(knex: Knex) {
+  await knex.schema.dropTable("recordings");
+}

src/db/migrations/20210311185912_move_duration_to_recordings_table.ts

@@ -0,0 +1,47 @@
+import { Knex } from "knex";
+
+export async function up(knex: Knex) {
+  return knex.transaction(async (trx) => {
+    await trx.schema.alterTable("recordings", (table) => {
+      table.integer("duration");
+    });
+
+    // transfer recording durations from voicemails table
+    await trx("recordings").update({
+      duration: knex("voicemails")
+        .select("duration")
+        .where("messageId", knex.raw("??", "recordings.messageId")),
+    });
+
+    // now we can make duration column not-nullable
+    await trx.schema.alterTable("recordings", (table) => {
+      table.integer("duration").notNullable().alter();
+    });
+
+    await trx.schema.alterTable("voicemails", (table) => {
+      table.dropColumn("duration");
+    });
+  });
+}
+
+export async function down(knex: Knex) {
+  return knex.transaction(async (trx) => {
+    await trx.schema.alterTable("voicemails", (table) => {
+      table.integer("duration");
+    });
+
+    await trx("voicemails").update({
+      duration: knex("recordings")
+        .select("duration")
+        .where("messageId", knex.raw("??", "voicemails.messageId")),
+    });
+
+    await trx.schema.alterTable("voicemails", (table) => {
+      table.integer("duration").notNullable().alter();
+    });
+
+    await trx.schema.alterTable("recordings", (table) => {
+      table.dropColumn("duration");
+    });
+  });
+}

src/index.ts

@@ -30,6 +30,22 @@ function getExtensionToTicketGroupMapping() {
   return mapping;
 }
 
+const DEFAULT_FIRST_RUN_AGE = 86400;
+
+function getTicketizeConfig() {
+  const firstRunAge = process.env.FIRST_RUN_AGE
+    ? parseInt(process.env.FIRST_RUN_AGE)
+    : DEFAULT_FIRST_RUN_AGE;
+  if (isNaN(firstRunAge) || firstRunAge <= 0) {
+    throw new Error("FIRST_RUN_AGE must be a valid positive integer");
+  }
+
+  return {
+    firstRunAge,
+    extensionToTicketGroup: getExtensionToTicketGroupMapping(),
+  };
+}
+
 async function initSonar() {
   const sonar = new Sonar(process.env.SONAR_URL!, process.env.SONAR_TOKEN!);
   // simple query to test API cedentials
@@ -52,6 +68,10 @@ async function initRingCentralSDK() {
     clientId: process.env.RC_APP_KEY,
     clientSecret: process.env.RC_APP_SECRET,
   });
+  const platform = sdk.platform();
+  platform.on(platform.events.refreshError, (err) => {
+    console.error(err);
+  });
   await sdk.login({
     username: process.env.RC_LOGIN_USERNAME,
     extension: process.env.RC_LOGIN_EXT,
@@ -79,9 +99,7 @@ async function main() {
     const db = await initDB();
 
     console.log("Starting ticketizer...");
-    const intervals = ticketize(sonar, rcsdk, db, {
+    const intervals = ticketize(sonar, rcsdk, db, getTicketizeConfig());
-      extensionToTicketGroup: getExtensionToTicketGroupMapping(),
-    });
 
     ["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => {
       process.on(sig, async () => {

src/template.tsx

@@ -1,8 +1,9 @@
 import React from "react";
 import ReactDOMServer from "react-dom/server";
-import type { Contact, StoredVoicemail } from "./types";
 import { getNationalNumber, formatSeconds } from "./util";
 import { DateTime } from "luxon";
+import type { Contact } from "./types";
+import type { StoredVoicemail, StoredRecording } from "knex/types/tables";
 
 export function getTicketSubject(
   voicemail: StoredVoicemail,
@@ -13,7 +14,10 @@ export function getTicketSubject(
   })`;
 }
 
-export function getTicketBody(vm: StoredVoicemail, contact?: Contact) {
+export function getTicketBody(
+  vm: StoredVoicemail & StoredRecording,
+  contact?: Contact
+) {
   return ReactDOMServer.renderToStaticMarkup(
     <div>
       <div>
@@ -33,7 +37,12 @@ export function getTicketBody(vm: StoredVoicemail, contact?: Contact) {
       </div>
       <br />
       <div>
-        <strong>Transcription: </strong>
+        <span>
+          <b>Transcription:</b>{" "}
+          {vm.transcriptionStatus === "CompletedPartially" ? (
+            <i>(partial)</i>
+          ) : undefined}
+        </span>
         <p>
           <i>
             {vm.transcription

src/ticketize.ts

@@ -11,8 +11,8 @@ import type {
   RCAudioAttachment,
   Recording,
   Transcription,
-  StoredVoicemail,
 } from "./types";
+import type { StoredVoicemail, StoredRecording } from "knex/types/tables";
 
 const SEARCH_CONTACT_BY_PHONE_NUMBER_QUERY = gql`
   query getContactByPhoneNumber($phoneNumber: String!) {
@@ -49,6 +49,7 @@ function rcapi(short: string, version = "v1.0") {
 }
 
 interface TicketizeConfig {
+  firstRunAge: number;
   extensionToTicketGroup: { [key: string]: number };
 }
 
@@ -62,7 +63,7 @@ export function ticketize(
   sonar: Sonar,
   rcsdk: SDK,
   db: Knex,
-  { extensionToTicketGroup }: TicketizeConfig
+  { firstRunAge, extensionToTicketGroup }: TicketizeConfig
 ) {
   /**
    * Uploads a file to Sonar, returning its ID.
@@ -92,17 +93,17 @@ export function ticketize(
   }
 
   /**
-   * Returns `extensionId`s messages that are up to `from` seconds old.
+   * Returns `extensionId`s messages that are up to `age` seconds old.
    *
    * @param extensionId
-   * @param from how many seconds ago to retrieve messages from
+   * @param age the maximum age (in seconds) of voicemails to fetch
    */
-  async function getExtensionVoicemails(extensionId: number, from = 86000) {
+  async function getExtensionVoicemails(extensionId: number, age = 86000) {
     const result = await rcsdk.get(
       rcapi(`/account/~/extension/${extensionId}/message-store`),
       {
         messageType: "VoiceMail",
-        dateFrom: new Date(Date.now() - from * 1000).toISOString(),
+        dateFrom: new Date(Date.now() - age * 1000).toISOString(),
       }
     );
     return (await result.json()).records as RCMessage[];
@@ -168,8 +169,8 @@ export function ticketize(
     const response = await rcsdk.get(audio.uri);
     const result = {
       duration: audio.vmDuration,
-      mimetype: audio.contentType,
+      mimeType: audio.contentType,
-      audio: await response.blob(),
+      audio: await response.arrayBuffer(),
     };
     return result;
   }
@@ -193,7 +194,10 @@ export function ticketize(
    * @param voicemail
    * @param contact
    */
-  async function createTicket(voicemail: StoredVoicemail, contact?: Contact) {
+  async function createTicket(
+    voicemail: StoredVoicemail & StoredRecording,
+    contact?: Contact
+  ) {
     const input: any = {
       subject: getTicketSubject(voicemail, contact),
       description: getTicketBody(voicemail, contact),
@@ -229,7 +233,8 @@ export function ticketize(
     recording: Recording,
     transcription: Transcription
   ) {
-    return db<StoredVoicemail>("voicemails").insert({
+    await db.transaction(async (trx) => {
+      await trx("voicemails").insert({
         messageId: message.id,
         extensionId: message.extensionId,
         received: message.creationTime,
@@ -238,20 +243,28 @@ export function ticketize(
         extensionName: extension.name,
         fromNumber: message.from.phoneNumber,
         fromName: message.from.name,
-      duration: recording.duration,
         transcriptionStatus: transcription.status,
         transcription: transcription.text,
       });
+      await trx("recordings").insert({
+        messageId: message.id,
+        mimeType: recording.mimeType,
+        audio: new Uint8Array(recording.audio),
+        duration: recording.duration,
+      });
+    });
   }
 
   /**
-   * Updates a stored voicemail using its current properties
+   * Updates a stored voicemail
    * @param voicemail the voicemail to update
    */
-  async function updateStoredVoicemail(voicemail: StoredVoicemail) {
+  async function updateStoredVoicemail(voicemail: Partial<StoredVoicemail>) {
-    await db<StoredVoicemail>("voicemails")
+    const messageId = voicemail.messageId;
-      .update({ ...voicemail })
+    if (!messageId) {
-      .where({ messageId: voicemail.messageId });
+      throw new Error("Missing required messageId property");
+    }
+    await db("voicemails").update(voicemail).where({ messageId });
   }
 
   /**
@@ -259,9 +272,7 @@ export function ticketize(
    * @returns whether the message by the given ID has been stored
    */
   async function isMessageStored(messageId: number) {
-    const result = await db<StoredVoicemail>("voicemails")
+    const result = await db("voicemails").where({ messageId }).first();
-      .where({ messageId })
-      .first();
     return result !== undefined;
   }
 
@@ -269,7 +280,8 @@ export function ticketize(
    * @returns stored voicemails that haven't had tickets created for them yet
    */
   async function getUnprocessedVoicemails() {
-    return await db<StoredVoicemail>("voicemails")
+    return await db("voicemails")
+      .join("recordings", "voicemails.messageId", "recordings.messageId")
       .whereNull("ticketId")
       .whereIn("transcriptionStatus", [
         "Completed",
@@ -283,7 +295,7 @@ export function ticketize(
    * @returns stored voicemails whose trranscriptions may still be in progress
    */
   async function getMissingTranscriptionVoicemails() {
-    return await db<StoredVoicemail>("voicemails")
+    return await db("voicemails")
       .whereNotNull("transcription")
       .whereNotIn("transcriptionStatus", [
         // Don't include those whose transcriptions have failed or will not
@@ -294,16 +306,13 @@ export function ticketize(
   }
 
   /**
-   * Retrieves and stores the voicemails for `extension` that are up to `from`
+   * Retrieves and stores the voicemails for `extension` that are up to `age`
    * seconds old.
    * @param extension
-   * @param from
+   * @param age
    */
-  async function storeExtensionVoicemails(
+  async function storeExtensionVoicemails(extension: RCExtension, age: number) {
-    extension: RCExtension,
+    const messages = await getExtensionVoicemails(extension.id, age);
-    from: number
-  ) {
-    const messages = await getExtensionVoicemails(extension.id, from);
     const isStored = await Promise.all(
       messages.map((message) => isMessageStored(message.id))
     );
@@ -324,15 +333,15 @@ export function ticketize(
 
   /**
    * Fetch and store new voicemails. If this is the first run, we get the last
-   * day's worth of voicemails. Otherwise, we fetch only the last 15 minutes.
+   * day's worth of voicemails. Otherwise, we fetch only the last 5 minutes.
    *
-   * @param firstRun whether this is the first run or not
+   * @param firstRun whether this is the first run
    */
   async function fetchAndStoreNewVoicemails(firstRun = false) {
     const extensions = await getValidRCExtensions();
     return Promise.all(
       extensions.map((extension) =>
-        storeExtensionVoicemails(extension, firstRun ? 86400 : 900)
+        storeExtensionVoicemails(extension, firstRun ? firstRunAge : 300)
       )
     );
   }
@@ -366,7 +375,11 @@ export function ticketize(
           // else we do nothing
           return;
         }
-        return updateStoredVoicemail(message);
+        return updateStoredVoicemail({
+          messageId: message.messageId,
+          transcriptionStatus: message.transcriptionStatus,
+          transcription: message.transcription,
+        });
       })
     );
   }
@@ -385,7 +398,7 @@ export function ticketize(
           `Created ticket ${ticketId} from voicemail ${voicemail.messageId}`
         );
         return updateStoredVoicemail({
-          ...voicemail,
+          messageId: voicemail.messageId,
           ticketId,
           contactId: contact?.id,
           contactableType: contact?.contactable.__typename,
@@ -403,22 +416,20 @@ export function ticketize(
   return [
     setAsyncInterval(
       () => {
-        const promise = fetchAndStoreNewVoicemails(firstRun);
+        const promise = fetchAndStoreNewVoicemails(firstRun).catch(
+          catchHandler
+        );
         firstRun = false;
-        return promise.catch(catchHandler);
+        return promise;
       },
       60 * 1000,
       true // immediate
     ),
     setAsyncInterval(
       () => fetchMissingTranscriptions().catch(catchHandler),
-      60 * 1000,
+      15 * 1000,
-      true
-    ),
-    setAsyncInterval(
-      () => createTickets().catch(catchHandler),
-      60 * 1000,
       true
     ),
+    setAsyncInterval(() => createTickets().catch(catchHandler), 1000, true),
   ];
 }

src/types.ts

@@ -97,8 +97,8 @@ export interface RCMessage {
 
 export interface Recording {
   duration: number;
-  mimetype: string;
+  mimeType: string;
-  audio: Blob;
+  audio: ArrayBuffer;
 }
 
 export interface Transcription {
@@ -106,17 +106,16 @@ export interface Transcription {
   text: string | null;
 }
 
-export interface StoredVoicemail {
+declare module "knex/types/tables" {
+  interface StoredVoicemail {
     messageId: number;
     extensionId: number;
-  processed: boolean;
     received: string;
     toNumber: string;
     extensionNumber: string;
     extensionName: string;
     fromNumber: string;
-  fromName?: string;
+    fromName: string;
-  duration: number;
     transcriptionStatus: TranscriptionStatus;
     transcription: string | null;
     ticketId?: number;
@@ -124,3 +123,16 @@ export interface StoredVoicemail {
     contactableType?: string;
     contactableId?: number;
   }
+
+  interface StoredRecording {
+    messageId: number;
+    mimeType: string;
+    audio: ArrayBuffer;
+    duration: number;
+  }
+
+  interface Tables {
+    voicemails: StoredVoicemail;
+    recordings: StoredRecording;
+  }
+}