chore: Add Redis PING healthcheck (#12157)

* chore: Add Redis PING healthcheck

* PR feedback

* fix incorrect reconnects

server/env.ts

@@ -13,6 +13,7 @@ import {
   IsNumber,
   IsIn,
   IsBoolean,
+  Min,
 } from "class-validator";
 import uniq from "lodash/uniq";
 import { languages } from "@shared/i18n";
@@ -199,6 +200,28 @@ export class Environment {
    */
   public REDIS_COLLABORATION_URL = environment.REDIS_COLLABORATION_URL;
 
+  /**
+   * The interval in milliseconds between redis connection healthchecks. Each
+   * healthcheck issues a PING and forces a reconnect if it fails.
+   */
+  @IsNumber()
+  @Min(1000)
+  public REDIS_HEALTHCHECK_INTERVAL = parseInt(
+    environment.REDIS_HEALTHCHECK_INTERVAL || "30000",
+    10
+  );
+
+  /**
+   * The timeout in milliseconds for a redis healthcheck PING before the
+   * connection is considered stuck and forcibly reconnected.
+   */
+  @IsNumber()
+  @Min(100)
+  public REDIS_HEALTHCHECK_TIMEOUT = parseInt(
+    environment.REDIS_HEALTHCHECK_TIMEOUT || "5000",
+    10
+  );
+
   /**
    * The fully qualified, external facing domain name of the server.
    * If not set, will be derived from HEROKU_APP_NAME for Heroku deployments.

server/storage/redis.ts

@@ -80,6 +80,37 @@ export default class RedisAdapter extends Redis {
         Logger.error("Redis error", err);
       }
     });
+
+    // Skip the healthcheck on connections reserved for blocking or pub/sub
+    // operations (signalled via maxRetriesPerRequest: null). A PING issued on
+    // those connections queues behind the in-flight blocking command and would
+    // spuriously time out.
+    if (this.options.maxRetriesPerRequest !== null) {
+      const healthcheck = setInterval(() => {
+        if (this.status !== "ready") {
+          return;
+        }
+
+        let pingTimeout: NodeJS.Timeout;
+        const timeoutPromise = new Promise((_, reject) => {
+          pingTimeout = setTimeout(
+            () => reject(new Error("ping timeout")),
+            env.REDIS_HEALTHCHECK_TIMEOUT
+          );
+        });
+
+        Promise.race([this.ping(), timeoutPromise])
+          .catch((err) => {
+            Logger.warn("Redis healthcheck failed, forcing reconnect", {
+              error: err,
+            });
+            this.disconnect(true);
+          })
+          .finally(() => clearTimeout(pingTimeout));
+      }, env.REDIS_HEALTHCHECK_INTERVAL);
+
+      this.on("end", () => clearInterval(healthcheck));
+    }
   }
 
   private static client: RedisAdapter;