feat: support custom elements in @excalidraw/excalidraw (#5164)

* feat: support custom elements in @excalidraw/excalidraw

* revert

* fix css

* fix offsets

* fix overflow of custom elements in example

* fix overflow in comments input

* make sure comment input never overflows the viewport

* remove offsetschange

* expose setActiveTool

* rename to onPointerDown

* update docs

* fix

src/components/App.tsx

@@ -384,6 +384,7 @@ class App extends React.Component<AppProps, AppState> {
         importLibrary: this.importLibraryFromUrl,
         setToastMessage: this.setToastMessage,
         id: this.id,
+        setActiveTool: this.setActiveTool,
       } as const;
       if (typeof excalidrawRef === "function") {
         excalidrawRef(api);
@@ -1059,6 +1060,13 @@ class App extends React.Component<AppProps, AppState> {
 
   componentDidUpdate(prevProps: AppProps, prevState: AppState) {
     if (
+      prevState.scrollX !== this.state.scrollX ||
+      prevState.scrollY !== this.state.scrollY
+    ) {
+      this.props?.onScrollChange?.(this.state.scrollX, this.state.scrollY);
+    }
+
+    if (
       Object.keys(this.state.selectedElementIds).length &&
       isEraserActive(this.state)
     ) {
@@ -1966,11 +1974,11 @@ class App extends React.Component<AppProps, AppState> {
     }
   });
 
-  private setActiveTool(
+  private setActiveTool = (
     tool:
       | { type: typeof SHAPES[number]["value"] | "eraser" }
       | { type: "custom"; customType: string },
-  ) {
+  ) => {
     const nextActiveTool = updateActiveTool(this.state, tool);
     if (!isHoldingSpace) {
       setCursorForShape(this.canvas, this.state);
@@ -1994,7 +2002,7 @@ class App extends React.Component<AppProps, AppState> {
     } else {
       this.setState({ activeTool: nextActiveTool });
     }
-  }
+  };
 
   private onGestureStart = withBatchedUpdates((event: GestureEvent) => {
     event.preventDefault();
@@ -3068,6 +3076,8 @@ class App extends React.Component<AppProps, AppState> {
       );
     }
 
+    this.props?.onPointerDown?.(this.state.activeTool, pointerDownState);
+
     const onPointerMove =
       this.onPointerMoveFromPointerDownHandler(pointerDownState);
 

src/packages/excalidraw/CHANGELOG.md

@@ -17,6 +17,12 @@ Please add the latest change on the top under the correct section.
 
 #### Features
 
+- Add support for integrating custom elements [#5164](https://github.com/excalidraw/excalidraw/pull/5164).
+
+  - Add [`onPointerDown`](https://github.com/excalidraw/excalidraw/blob/master/src/packages/excalidraw/README.md#onPointerDown) callback which gets triggered on pointer down events.
+  - Add [`onScrollChange`](https://github.com/excalidraw/excalidraw/blob/master/src/packages/excalidraw/README.md#onScrollChange) callback which gets triggered when scrolling the canvas.
+  - Add API [`setActiveTool`](https://github.com/excalidraw/excalidraw/blob/master/src/packages/excalidraw/README.md#setActiveTool) which host can call to set the active tool.
+
 - Exported [`loadSceneOrLibraryFromBlob`](https://github.com/excalidraw/excalidraw/blob/master/src/packages/excalidraw/README.md#loadSceneOrLibraryFromBlob) function [#5057](https://github.com/excalidraw/excalidraw/pull/5057).
 - Export [`MIME_TYPES`](https://github.com/excalidraw/excalidraw/blob/master/src/constants.ts#L92) supported by Excalidraw [#5135](https://github.com/excalidraw/excalidraw/pull/5135).
 - Support [`src`](https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L50) for collaborators. Now onwards host can pass `src` to render the customized avatar for collaborators [#5114](https://github.com/excalidraw/excalidraw/pull/5114).

src/packages/excalidraw/README_NEXT.md

@@ -405,7 +405,9 @@ For a complete list of variables, check [theme.scss](https://github.com/excalidr
 | [`onLibraryChange`](#onLibraryChange) | <pre>(items: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L200">LibraryItems</a>) => void &#124; Promise&lt;any&gt; </pre> |  | The callback if supplied is triggered when the library is updated and receives the library items. |
 | [`autoFocus`](#autoFocus) | boolean | false | Implies whether to focus the Excalidraw component on page load |
 | [`generateIdForFile`](#generateIdForFile) | `(file: File) => string | Promise<string>` | Allows you to override `id` generation for files added on canvas |
-| [`onLinkOpen`](#onLinkOpen) | <pre>(element: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/element/types.ts#L106">NonDeletedExcalidrawElement</a>, event: CustomEvent) </pre> |  | This prop if passed will be triggered when link of an element is clicked |
+| [`onLinkOpen`](#onLinkOpen) | <pre>(element: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/element/types.ts#L106">NonDeletedExcalidrawElement</a>, event: CustomEvent) </pre> |  | This prop if passed will be triggered when link of an element is clicked. |
+| [`onPointerDown`](#onPointerDown) | <pre>(activeTool: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L93"> AppState["activeTool"]</a>, pointerDownState: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L365">PointerDownState</a>) => void</pre> |  | This prop if passed gets triggered on pointer down evenets |
+| [`onScrollChange`](#onScrollChange) | (scrollX: number, scrollY: number) |  | This prop if passed gets triggered when scrolling the canvas. |
 
 ### Dimensions of Excalidraw
 
@@ -491,6 +493,7 @@ You can pass a `ref` when you want to access some excalidraw APIs. We expose the
 | setToastMessage | `(message: string) => void` | This API can be used to show the toast with custom message. |
 | [id](#id) | string | Unique ID for the excalidraw component. |
 | [getFiles](#getFiles) | <pre>() => <a href="https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L64">files</a> </pre> | This API can be used to get the files present in the scene. It may contain files that aren't referenced by any element, so if you're persisting the files to a storage, you should compare them against stored elements. |
+| [setActiveTool](#setActiveTool) | <pre>(tool: { type: typeof <a href="https://github.com/excalidraw/excalidraw/blob/master/src/shapes.tsx#L4">SHAPES</a>[number]["value"] &#124; "eraser" } &#124; { type: "custom"; customType: string }) => void</pre> | This API can be used to set the active tool |
 
 #### `readyPromise`
 
@@ -673,6 +676,14 @@ useEffect(() => {
 
 Try out the [Demo](#Demo) to see it in action.
 
+#### `setActiveTool`
+
+This API has the below signature. It sets the `tool` passed in param as the active tool.
+
+<pre>
+(tool: { type: typeof <a href="https://github.com/excalidraw/excalidraw/blob/master/src/shapes.tsx#L4">SHAPES</a>[number]["value"] &#124; "eraser" } &#124; { type: "custom"; customType: string }) => void
+</pre>
+
 #### `detectScroll`
 
 Indicates whether Excalidraw should listen for `scroll` event on the nearest scrollable container in the DOM tree and recompute the coordinates (e.g. to correctly handle the cursor) when the component's position changes. You can disable this when you either know this doesn't affect your app or you want to take care of it yourself (calling the [`refresh()`](#ref) method).
@@ -742,6 +753,22 @@ const onLinkOpen: ExcalidrawProps["onLinkOpen"] = useCallback(
 );
 ```
 
+#### `onPointerDown`
+
+This prop if passed will be triggered on pointer down events and has the below signature.
+
+<pre>
+(activeTool: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L93"> AppState["activeTool"]</a>, pointerDownState: <a href="https://github.com/excalidraw/excalidraw/blob/master/src/types.ts#L365">PointerDownState</a>) => void
+</pre>
+
+#### `onScrollChange`
+
+This prop if passed will be triggered when canvas is scrolled and has the below signature.
+
+```ts
+(scrollX: number, scrollY: number) => void
+```
+
 ### Does it support collaboration ?
 
 No, Excalidraw package doesn't come with collaboration built in, since the implementation is specific to each host app. We expose APIs which you can use to communicate with Excalidraw which you can use to implement it. You can check our own implementation [here](https://github.com/excalidraw/excalidraw/blob/master/src/excalidraw-app/index.tsx).

src/packages/excalidraw/example/App.js

@@ -5,6 +5,15 @@ import Sidebar from "./sidebar/Sidebar";
 
 import "./App.scss";
 import initialData from "./initialData";
+import { nanoid } from "nanoid";
+import {
+  sceneCoordsToViewportCoords,
+  viewportCoordsToSceneCoords,
+  withBatchedUpdates,
+  withBatchedUpdatesThrottled,
+} from "../../../utils";
+import { DRAGGING_THRESHOLD, EVENT } from "../../../constants";
+import { distance2d } from "../../../math";
 import { fileOpen } from "../../../data/filesystem";
 import { loadSceneOrLibraryFromBlob } from "../../utils";
 
@@ -20,6 +29,15 @@ const {
   MIME_TYPES,
 } = window.ExcalidrawLib;
 
+const COMMENT_SVG = (
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
+    <path d="M256 32C114.6 32 .0272 125.1 .0272 240c0 47.63 19.91 91.25 52.91 126.2c-14.88 39.5-45.87 72.88-46.37 73.25c-6.625 7-8.375 17.25-4.625 26C5.818 474.2 14.38 480 24 480c61.5 0 109.1-25.75 139.1-46.25C191.1 442.8 223.3 448 256 448c141.4 0 255.1-93.13 255.1-208S397.4 32 256 32zM256.1 400c-26.75 0-53.12-4.125-78.38-12.12l-22.75-7.125l-19.5 13.75c-14.25 10.12-33.88 21.38-57.5 29c7.375-12.12 14.37-25.75 19.88-40.25l10.62-28l-20.62-21.87C69.82 314.1 48.07 282.2 48.07 240c0-88.25 93.25-160 208-160s208 71.75 208 160S370.8 400 256.1 400z" />
+  </svg>
+);
+const COMMENT_ICON_DIMENSION = 32;
+const COMMENT_INPUT_HEIGHT = 50;
+const COMMENT_INPUT_WIDTH = 150;
+
 const resolvablePromise = () => {
   let resolve;
   let reject;
@@ -44,17 +62,9 @@ const renderTopRightUI = () => {
   );
 };
 
-const renderFooter = () => {
-  return (
-    <button onClick={() => alert("This is dummy footer")}>
-      {" "}
-      custom footer{" "}
-    </button>
-  );
-};
-
 export default function App() {
   const excalidrawRef = useRef(null);
+  const appRef = useRef(null);
 
   const [viewModeEnabled, setViewModeEnabled] = useState(false);
   const [zenModeEnabled, setZenModeEnabled] = useState(false);
@@ -65,6 +75,8 @@ export default function App() {
   const [exportEmbedScene, setExportEmbedScene] = useState(false);
   const [theme, setTheme] = useState("light");
   const [isCollaborating, setIsCollaborating] = useState(false);
+  const [commentIcons, setCommentIcons] = useState({});
+  const [comment, setComment] = useState(null);
 
   const initialStatePromiseRef = useRef({ promise: null });
   if (!initialStatePromiseRef.current.promise) {
@@ -105,6 +117,28 @@ export default function App() {
       window.removeEventListener("hashchange", onHashChange);
     };
   }, []);
+  const renderFooter = () => {
+    return (
+      <>
+        {" "}
+        <button
+          className="custom-element"
+          onClick={() =>
+            excalidrawRef.current.setActiveTool({
+              type: "custom",
+              customType: "comment",
+            })
+          }
+        >
+          {COMMENT_SVG}
+        </button>
+        <button onClick={() => alert("This is dummy footer")}>
+          {" "}
+          custom footer{" "}
+        </button>
+      </>
+    );
+  };
 
   const loadSceneOrLibrary = async () => {
     const file = await fileOpen({ description: "Excalidraw or library file" });
@@ -168,8 +202,210 @@ export default function App() {
     });
     window.alert(`Copied to clipboard as ${type} sucessfully`);
   };
+
+  const onPointerDown = (activeTool, pointerDownState) => {
+    if (activeTool.type === "custom" && activeTool.customType === "comment") {
+      const { x, y } = pointerDownState.origin;
+      setComment({ x, y, value: "" });
+    }
+  };
+
+  const rerenderCommentIcons = () => {
+    const commentIconsElements =
+      appRef.current.querySelectorAll(".comment-icon");
+    commentIconsElements.forEach((ele) => {
+      const id = ele.id;
+      const appstate = excalidrawRef.current.getAppState();
+      const { x, y } = sceneCoordsToViewportCoords(
+        { sceneX: commentIcons[id].x, sceneY: commentIcons[id].y },
+        appstate,
+      );
+      ele.style.left = `${
+        x - COMMENT_ICON_DIMENSION / 2 - appstate.offsetLeft
+      }px`;
+      ele.style.top = `${
+        y - COMMENT_ICON_DIMENSION / 2 - appstate.offsetTop
+      }px`;
+    });
+  };
+
+  const onPointerMoveFromPointerDownHandler = (pointerDownState) => {
+    return withBatchedUpdatesThrottled((event) => {
+      const { x, y } = viewportCoordsToSceneCoords(
+        { clientX: event.clientX, clientY: event.clientY },
+        excalidrawRef.current.getAppState(),
+      );
+      const distance = distance2d(
+        pointerDownState.x,
+        pointerDownState.y,
+        event.clientX,
+        event.clientY,
+      );
+      if (distance > DRAGGING_THRESHOLD) {
+        setCommentIcons({
+          ...commentIcons,
+          [pointerDownState.hitElement.id]: {
+            ...commentIcons[pointerDownState.hitElement.id],
+            x,
+            y,
+          },
+        });
+      }
+    });
+  };
+  const onPointerUpFromPointerDownHandler = (pointerDownState) => {
+    return withBatchedUpdates((event) => {
+      window.removeEventListener(EVENT.POINTER_MOVE, pointerDownState.onMove);
+      window.removeEventListener(EVENT.POINTER_UP, pointerDownState.onUp);
+      excalidrawRef.current.setActiveTool({ type: "selection" });
+      const distance = distance2d(
+        pointerDownState.x,
+        pointerDownState.y,
+        event.clientX,
+        event.clientY,
+      );
+      if (distance === 0) {
+        if (!comment) {
+          setComment({
+            x: pointerDownState.hitElement.x + 60,
+            y: pointerDownState.hitElement.y,
+            value: pointerDownState.hitElement.value,
+            id: pointerDownState.hitElement.id,
+          });
+        } else {
+          setComment(null);
+        }
+      }
+    });
+  };
+  const renderCommentIcons = () => {
+    return Object.values(commentIcons).map((commentIcon) => {
+      const appState = excalidrawRef.current.getAppState();
+      const { x, y } = sceneCoordsToViewportCoords(
+        { sceneX: commentIcon.x, sceneY: commentIcon.y },
+        excalidrawRef.current.getAppState(),
+      );
+      return (
+        <div
+          id={commentIcon.id}
+          key={commentIcon.id}
+          style={{
+            top: `${y - COMMENT_ICON_DIMENSION / 2 - appState.offsetTop}px`,
+            left: `${x - COMMENT_ICON_DIMENSION / 2 - appState.offsetLeft}px`,
+            position: "absolute",
+            zIndex: 1,
+            width: `${COMMENT_ICON_DIMENSION}px`,
+            height: `${COMMENT_ICON_DIMENSION}px`,
+          }}
+          className="comment-icon"
+          onPointerDown={(event) => {
+            event.preventDefault();
+            if (comment) {
+              commentIcon.value = comment.value;
+              saveComment();
+            }
+            const pointerDownState = {
+              x: event.clientX,
+              y: event.clientY,
+              hitElement: commentIcon,
+            };
+            const onPointerMove =
+              onPointerMoveFromPointerDownHandler(pointerDownState);
+            const onPointerUp =
+              onPointerUpFromPointerDownHandler(pointerDownState);
+            window.addEventListener(EVENT.POINTER_MOVE, onPointerMove);
+            window.addEventListener(EVENT.POINTER_UP, onPointerUp);
+
+            pointerDownState.onMove = onPointerMove;
+            pointerDownState.onUp = onPointerUp;
+
+            excalidrawRef.current.setCustomType("comment");
+          }}
+        >
+          <div className="comment-avatar">
+            <img src="doremon.png" alt="doremon" />
+          </div>
+        </div>
+      );
+    });
+  };
+
+  const saveComment = () => {
+    if (!comment.id && !comment.value) {
+      setComment(null);
+      return;
+    }
+    const id = comment.id || nanoid();
+    setCommentIcons({
+      ...commentIcons,
+      [id]: {
+        x: comment.id ? comment.x - 60 : comment.x,
+        y: comment.y,
+        id,
+        value: comment.value,
+      },
+    });
+    setComment(null);
+  };
+
+  const renderComment = () => {
+    const appState = excalidrawRef.current.getAppState();
+    const { x, y } = sceneCoordsToViewportCoords(
+      { sceneX: comment.x, sceneY: comment.y },
+      appState,
+    );
+    let top = y - COMMENT_ICON_DIMENSION / 2 - appState.offsetTop;
+    let left = x - COMMENT_ICON_DIMENSION / 2 - appState.offsetLeft;
+
+    if (
+      top + COMMENT_INPUT_HEIGHT <
+      appState.offsetTop + COMMENT_INPUT_HEIGHT
+    ) {
+      top = COMMENT_ICON_DIMENSION / 2;
+    }
+    if (top + COMMENT_INPUT_HEIGHT > appState.height) {
+      top = appState.height - COMMENT_INPUT_HEIGHT - COMMENT_ICON_DIMENSION / 2;
+    }
+    if (
+      left + COMMENT_INPUT_WIDTH <
+      appState.offsetLeft + COMMENT_INPUT_WIDTH
+    ) {
+      left = COMMENT_ICON_DIMENSION / 2;
+    }
+    if (left + COMMENT_INPUT_WIDTH > appState.width) {
+      left = appState.width - COMMENT_INPUT_WIDTH - COMMENT_ICON_DIMENSION / 2;
+    }
+    return (
+      <textarea
+        className="comment"
+        style={{
+          top: `${top}px`,
+          left: `${left}px`,
+          position: "absolute",
+          zIndex: 1,
+          height: `${COMMENT_INPUT_HEIGHT}px`,
+          width: `${COMMENT_INPUT_WIDTH}px`,
+        }}
+        ref={(ref) => {
+          setTimeout(() => ref?.focus());
+        }}
+        placeholder={comment.value ? "Reply" : "Comment"}
+        value={comment.value}
+        onChange={(event) => {
+          setComment({ ...comment, value: event.target.value });
+        }}
+        onBlur={saveComment}
+        onKeyDown={(event) => {
+          if (!event.shiftKey && event.key === "Enter") {
+            event.preventDefault();
+            saveComment();
+          }
+        }}
+      />
+    );
+  };
   return (
-    <div className="App">
+    <div className="App" ref={appRef}>
       <h1> Excalidraw Example</h1>
       <Sidebar>
         <div className="button-wrapper">
@@ -288,9 +524,9 @@ export default function App() {
           <Excalidraw
             ref={excalidrawRef}
             initialData={initialStatePromiseRef.current.promise}
-            onChange={(elements, state) =>
-              console.info("Elements :", elements, "State : ", state)
-            }
+            onChange={(elements, state) => {
+              console.info("Elements :", elements, "State : ", state);
+            }}
             onPointerUpdate={(payload) => console.info(payload)}
             onCollabButtonClick={() =>
               window.alert("You clicked on collab button")
@@ -304,7 +540,11 @@ export default function App() {
             renderTopRightUI={renderTopRightUI}
             renderFooter={renderFooter}
             onLinkOpen={onLinkOpen}
+            onPointerDown={onPointerDown}
+            onScrollChange={rerenderCommentIcons}
           />
+          {Object.keys(commentIcons || []).length > 0 && renderCommentIcons()}
+          {comment && renderComment()}
         </div>
 
         <div className="export-wrapper button-wrapper">
@@ -338,7 +578,8 @@ export default function App() {
                 embedScene: true,
                 files: excalidrawRef.current.getFiles(),
               });
-              document.querySelector(".export-svg").innerHTML = svg.outerHTML;
+              appRef.current.querySelector(".export-svg").innerHTML =
+                svg.outerHTML;
             }}
           >
             Export to SVG

src/packages/excalidraw/example/App.scss

@@ -1,6 +1,20 @@
 .App {
   font-family: sans-serif;
   text-align: center;
+
+  .comment-avatar {
+    background: #faa2c1;
+    border-radius: 66px 67px 67px 0px;
+    width: 2rem;
+    height: 2rem;
+    padding: 4px;
+    margin: 4px;
+    img {
+      width: 100%;
+      height: 100%;
+      border-radius: 50%;
+    }
+  }
 }
 
 .button-wrapper button {
@@ -18,6 +32,8 @@
 .excalidraw-wrapper {
   height: 800px;
   margin: 50px;
+  position: relative;
+  overflow: hidden;
 }
 
 :root[dir="ltr"]
@@ -46,4 +62,14 @@
   --color-primary-darker: #f783ac;
   --color-primary-darkest: #e64980;
   --color-primary-light: #fcc2d7;
+
+  button.custom-element {
+    width: 2rem;
+    height: 2rem;
+    margin: 0.4rem;
+    margin-left: -10px;
+  }
+  .layer-ui__wrapper__footer-center {
+    display: flex;
+  }
 }

src/packages/excalidraw/index.tsx

@@ -37,6 +37,8 @@ const ExcalidrawBase = (props: ExcalidrawProps) => {
     autoFocus = false,
     generateIdForFile,
     onLinkOpen,
+    onPointerDown,
+    onScrollChange,
   } = props;
 
   const canvasActions = props.UIOptions?.canvasActions;
@@ -100,6 +102,8 @@ const ExcalidrawBase = (props: ExcalidrawProps) => {
           autoFocus={autoFocus}
           generateIdForFile={generateIdForFile}
           onLinkOpen={onLinkOpen}
+          onPointerDown={onPointerDown}
+          onScrollChange={onScrollChange}
         />
       </Provider>
     </InitializeApp>

src/types.ts

@@ -289,6 +289,11 @@ export interface ExcalidrawProps {
       nativeEvent: MouseEvent | React.PointerEvent<HTMLCanvasElement>;
     }>,
   ) => void;
+  onPointerDown?: (
+    activeTool: AppState["activeTool"],
+    pointerDownState: PointerDownState,
+  ) => void;
+  onScrollChange?: (scrollX: number, scrollY: number) => void;
 }
 
 export type SceneData = {
@@ -449,6 +454,7 @@ export type ExcalidrawImperativeAPI = {
   readyPromise: ResolvablePromise<ExcalidrawImperativeAPI>;
   ready: true;
   id: string;
+  setActiveTool: InstanceType<typeof App>["setActiveTool"];
 };
 
 export type DeviceType = {