feat: Add idle detection to collaboration feature (#2877)

* Start idle detection implementation

* First working version

* Add screen state

* Add type safety

* Better rendering, enum types, localization

* Add origin trial token

* Fix

* Refactor idle detection to no longer use IdleDetector API

* Cleanup some leftovers

* Fix

* Apply suggestions from code review

* Three state: active 🟢, idle 💤, away ⚫️

* Address feedback from code review
Thanks, @lipis

* Deal with unmount

Co-authored-by: Panayiotis Lipiridis <lipiridis@gmail.com>

public/index.html

@@ -57,6 +57,7 @@
 
     <!-- Excalidraw version -->
     <meta name="version" content="{version}" />
+
     <link
       rel="preload"
       href="FG_Virgil.woff2"

src/components/App.tsx

@@ -882,6 +882,7 @@ class App extends React.Component<ExcalidrawProps, AppState> {
     const pointerViewportCoords: SceneState["remotePointerViewportCoords"] = {};
     const remoteSelectedElementIds: SceneState["remoteSelectedElementIds"] = {};
     const pointerUsernames: { [id: string]: string } = {};
+    const pointerUserStates: { [id: string]: string } = {};
     this.state.collaborators.forEach((user, socketId) => {
       if (user.selectedElementIds) {
         for (const id of Object.keys(user.selectedElementIds)) {
@@ -897,6 +898,9 @@ class App extends React.Component<ExcalidrawProps, AppState> {
       if (user.username) {
         pointerUsernames[socketId] = user.username;
       }
+      if (user.userState) {
+        pointerUserStates[socketId] = user.userState;
+      }
       pointerViewportCoords[socketId] = sceneCoordsToViewportCoords(
         {
           sceneX: user.pointer.x,
@@ -931,6 +935,7 @@ class App extends React.Component<ExcalidrawProps, AppState> {
         remotePointerButton: cursorButton,
         remoteSelectedElementIds,
         remotePointerUsernames: pointerUsernames,
+        remotePointerUserStates: pointerUserStates,
         shouldCacheIgnoreZoom: this.state.shouldCacheIgnoreZoom,
       },
       {

src/constants.ts

@@ -46,6 +46,7 @@ export enum EVENT {
   TOUCH_START = "touchstart",
   TOUCH_END = "touchend",
   HASHCHANGE = "hashchange",
+  VISIBILITY_CHANGE = "visibilitychange",
 }
 
 export const ENV = {
@@ -93,3 +94,8 @@ export const TOAST_TIMEOUT = 5000;
 export const VERSION_TIMEOUT = 30000;
 
 export const ZOOM_STEP = 0.1;
+
+// Report a user inactive after IDLE_THRESHOLD milliseconds
+export const IDLE_THRESHOLD = 60_000;
+// Report a user active each ACTIVE_THRESHOLD milliseconds
+export const ACTIVE_THRESHOLD = 3_000;

src/excalidraw-app/collab/CollabWrapper.tsx

@@ -38,11 +38,14 @@ import Portal from "./Portal";
 import RoomDialog from "./RoomDialog";
 import { createInverseContext } from "../../createInverseContext";
 import { t } from "../../i18n";
+import { UserIdleState } from "./types";
+import { IDLE_THRESHOLD, ACTIVE_THRESHOLD } from "../../constants";
 
 interface CollabState {
   modalIsShown: boolean;
   errorMessage: string;
   username: string;
+  userState: UserIdleState;
   activeRoomLink: string;
 }
 
@@ -52,6 +55,7 @@ export interface CollabAPI {
   /** function so that we can access the latest value from stale callbacks */
   isCollaborating: () => boolean;
   username: CollabState["username"];
+  userState: CollabState["userState"];
   onPointerUpdate: CollabInstance["onPointerUpdate"];
   initializeSocketClient: CollabInstance["initializeSocketClient"];
   onCollabButtonClick: CollabInstance["onCollabButtonClick"];
@@ -78,6 +82,8 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
   portal: Portal;
   excalidrawAPI: Props["excalidrawAPI"];
   isCollaborating: boolean = false;
+  activeIntervalId: number | null;
+  idleTimeoutId: number | null;
 
   private socketInitializationTimer?: NodeJS.Timeout;
   private lastBroadcastedOrReceivedSceneVersion: number = -1;
@@ -89,10 +95,13 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
       modalIsShown: false,
       errorMessage: "",
       username: importUsernameFromLocalStorage() || "",
+      userState: UserIdleState.ACTIVE,
       activeRoomLink: "",
     };
     this.portal = new Portal(this);
     this.excalidrawAPI = props.excalidrawAPI;
+    this.activeIntervalId = null;
+    this.idleTimeoutId = null;
   }
 
   componentDidMount() {
@@ -116,6 +125,19 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
   componentWillUnmount() {
     window.removeEventListener(EVENT.BEFORE_UNLOAD, this.beforeUnload);
     window.removeEventListener(EVENT.UNLOAD, this.onUnload);
+    window.removeEventListener(EVENT.POINTER_MOVE, this.onPointerMove);
+    window.removeEventListener(
+      EVENT.VISIBILITY_CHANGE,
+      this.onVisibilityChange,
+    );
+    if (this.activeIntervalId) {
+      window.clearInterval(this.activeIntervalId);
+      this.activeIntervalId = null;
+    }
+    if (this.idleTimeoutId) {
+      window.clearTimeout(this.idleTimeoutId);
+      this.idleTimeoutId = null;
+    }
   }
 
   private onUnload = () => {
@@ -318,6 +340,17 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
             });
             break;
           }
+          case "IDLE_STATUS": {
+            const { userState, socketId, username } = decryptedData.payload;
+            const collaborators = new Map(this.collaborators);
+            const user = collaborators.get(socketId) || {}!;
+            user.userState = userState;
+            user.username = username;
+            this.excalidrawAPI.updateScene({
+              collaborators,
+            });
+            break;
+          }
         }
       },
     );
@@ -330,6 +363,8 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
       scenePromise.resolve(null);
     });
 
+    this.initializeIdleDetector();
+
     this.setState({
       activeRoomLink: window.location.href,
     });
@@ -398,7 +433,7 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
     // Avoid broadcasting to the rest of the collaborators the scene
     // we just received!
     // Note: this needs to be set before updating the scene as it
-    // syncronously calls render.
+    // synchronously calls render.
     this.setLastBroadcastedOrReceivedSceneVersion(getSceneVersion(newElements));
 
     return newElements as ReconciledElements;
@@ -427,6 +462,58 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
     this.excalidrawAPI.history.clear();
   };
 
+  private onPointerMove = () => {
+    if (this.idleTimeoutId) {
+      window.clearTimeout(this.idleTimeoutId);
+      this.idleTimeoutId = null;
+    }
+    this.idleTimeoutId = window.setTimeout(this.reportIdle, IDLE_THRESHOLD);
+    if (!this.activeIntervalId) {
+      this.activeIntervalId = window.setInterval(
+        this.reportActive,
+        ACTIVE_THRESHOLD,
+      );
+    }
+  };
+
+  private onVisibilityChange = () => {
+    if (document.hidden) {
+      if (this.idleTimeoutId) {
+        window.clearTimeout(this.idleTimeoutId);
+        this.idleTimeoutId = null;
+      }
+      if (this.activeIntervalId) {
+        window.clearInterval(this.activeIntervalId);
+        this.activeIntervalId = null;
+      }
+      this.onIdleStateChange(UserIdleState.AWAY);
+    } else {
+      this.idleTimeoutId = window.setTimeout(this.reportIdle, IDLE_THRESHOLD);
+      this.activeIntervalId = window.setInterval(
+        this.reportActive,
+        ACTIVE_THRESHOLD,
+      );
+      this.onIdleStateChange(UserIdleState.ACTIVE);
+    }
+  };
+
+  private reportIdle = () => {
+    this.onIdleStateChange(UserIdleState.IDLE);
+    if (this.activeIntervalId) {
+      window.clearInterval(this.activeIntervalId);
+      this.activeIntervalId = null;
+    }
+  };
+
+  private reportActive = () => {
+    this.onIdleStateChange(UserIdleState.ACTIVE);
+  };
+
+  private initializeIdleDetector = () => {
+    document.addEventListener(EVENT.POINTER_MOVE, this.onPointerMove);
+    document.addEventListener(EVENT.VISIBILITY_CHANGE, this.onVisibilityChange);
+  };
+
   setCollaborators(sockets: string[]) {
     this.setState((state) => {
       const collaborators: InstanceType<
@@ -466,6 +553,11 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
       this.portal.broadcastMouseLocation(payload);
   };
 
+  onIdleStateChange = (userState: UserIdleState) => {
+    this.setState({ userState });
+    this.portal.broadcastIdleChange(userState);
+  };
+
   broadcastElements = (elements: readonly ExcalidrawElement[]) => {
     if (
       getSceneVersion(elements) >

src/excalidraw-app/collab/Portal.tsx

@@ -9,6 +9,7 @@ import CollabWrapper from "./CollabWrapper";
 import { getSyncableElements } from "../../packages/excalidraw/index";
 import { ExcalidrawElement } from "../../element/types";
 import { BROADCAST, SCENE } from "../app_constants";
+import { UserIdleState } from "./types";
 
 class Portal {
   collab: CollabWrapper;
@@ -132,6 +133,23 @@ class Portal {
     }
   };
 
+  broadcastIdleChange = (userState: UserIdleState) => {
+    if (this.socket?.id) {
+      const data: SocketUpdateDataSource["IDLE_STATUS"] = {
+        type: "IDLE_STATUS",
+        payload: {
+          socketId: this.socket.id,
+          userState,
+          username: this.collab.state.username,
+        },
+      };
+      return this._broadcastSocketData(
+        data as SocketUpdateData,
+        true, // volatile
+      );
+    }
+  };
+
   broadcastMouseLocation = (payload: {
     pointer: SocketUpdateDataSource["MOUSE_LOCATION"]["payload"]["pointer"];
     button: SocketUpdateDataSource["MOUSE_LOCATION"]["payload"]["button"];

src/excalidraw-app/collab/types.ts

@@ -0,0 +1,5 @@
+export enum UserIdleState {
+  ACTIVE = "active",
+  AWAY = "away",
+  IDLE = "idle",
+}

src/excalidraw-app/data/index.ts

@@ -4,6 +4,7 @@ import { ImportedDataState } from "../../data/types";
 import { ExcalidrawElement } from "../../element/types";
 import { t } from "../../i18n";
 import { AppState } from "../../types";
+import { UserIdleState } from "../collab/types";
 
 const byteToHex = (byte: number): string => `0${byte.toString(16)}`.slice(-2);
 
@@ -59,6 +60,14 @@ export type SocketUpdateDataSource = {
       username: string;
     };
   };
+  IDLE_STATUS: {
+    type: "IDLE_STATUS";
+    payload: {
+      socketId: string;
+      userState: UserIdleState;
+      username: string;
+    };
+  };
 };
 
 export type SocketUpdateDataIncoming =

src/renderer/renderScene.ts

@@ -48,6 +48,7 @@ import {
   TransformHandleType,
 } from "../element/transformHandles";
 import { viewportCoordsToSceneCoords } from "../utils";
+import { UserIdleState } from "../excalidraw-app/collab/types";
 
 const strokeRectWithRotation = (
   context: CanvasRenderingContext2D,
@@ -445,7 +446,9 @@ export const renderScene = (
     const globalAlpha = context.globalAlpha;
     context.strokeStyle = stroke;
     context.fillStyle = background;
-    if (isOutOfBounds) {
+
+    const userState = sceneState.remotePointerUserStates[clientId];
+    if (isOutOfBounds || userState === UserIdleState.AWAY) {
       context.globalAlpha = 0.2;
     }
 
@@ -478,19 +481,25 @@ export const renderScene = (
     context.stroke();
 
     const username = sceneState.remotePointerUsernames[clientId];
-
-    if (!isOutOfBounds && username) {
+    const usernameAndIdleState = `${username ? `${username} ` : ""}${
+      userState === UserIdleState.AWAY
+        ? "⚫️"
+        : userState === UserIdleState.IDLE
+        ? "💤"
+        : "🟢"
+    }`;
+
+    if (!isOutOfBounds && usernameAndIdleState) {
       const offsetX = x + width;
       const offsetY = y + height;
       const paddingHorizontal = 4;
       const paddingVertical = 4;
-      const measure = context.measureText(username);
+      const measure = context.measureText(usernameAndIdleState);
       const measureHeight =
         measure.actualBoundingBoxDescent + measure.actualBoundingBoxAscent;
 
       // Border
       context.fillStyle = stroke;
-      context.globalAlpha = globalAlpha;
       context.fillRect(
         offsetX - 1,
         offsetY - 1,
@@ -506,8 +515,9 @@ export const renderScene = (
         measureHeight + 2 * paddingVertical,
       );
       context.fillStyle = oc.white;
+
       context.fillText(
-        username,
+        usernameAndIdleState,
         offsetX + paddingHorizontal,
         offsetY + paddingVertical + measure.actualBoundingBoxAscent,
       );

src/scene/export.ts

@@ -65,6 +65,7 @@ export const exportToCanvas = (
       remoteSelectedElementIds: {},
       shouldCacheIgnoreZoom: false,
       remotePointerUsernames: {},
+      remotePointerUserStates: {},
     },
     {
       renderScrollbars: false,

src/scene/types.ts

@@ -12,6 +12,7 @@ export type SceneState = {
   remotePointerButton?: { [id: string]: string | undefined };
   remoteSelectedElementIds: { [elementId: string]: string[] };
   remotePointerUsernames: { [id: string]: string };
+  remotePointerUserStates: { [id: string]: string };
 };
 
 export type SceneScroll = {

src/types.ts

@@ -20,6 +20,7 @@ import { ExcalidrawImperativeAPI } from "./components/App";
 import type { ResolvablePromise } from "./utils";
 import { Spreadsheet } from "./charts";
 import { Language } from "./i18n";
+import { UserIdleState } from "./excalidraw-app/collab/types";
 
 export type Point = Readonly<RoughPoint>;
 
@@ -31,6 +32,7 @@ export type Collaborator = {
   button?: "up" | "down";
   selectedElementIds?: AppState["selectedElementIds"];
   username?: string | null;
+  userState?: UserIdleState;
 };
 
 export type AppState = {