Show diff by sentence (#477)

components/custom/diffview.tsx

@@ -13,7 +13,7 @@ import React, { useEffect, useRef } from 'react';
 import { renderToString } from 'react-dom/server';
 import ReactMarkdown from 'react-markdown';
 
-import { diffEditor, DiffType } from '@/lib/editor/index';
+import { diffEditor, DiffType } from '@/lib/editor/diff';
 
 const diffSchema = new Schema({
   nodes: addListNodes(schema.spec.nodes, 'paragraph block*', 'block'),

components/custom/editor.tsx

@@ -8,7 +8,7 @@ import { schema } from 'prosemirror-schema-basic';
 import { addListNodes } from 'prosemirror-schema-list';
 import { EditorState } from 'prosemirror-state';
 import { Decoration, DecorationSet, EditorView } from 'prosemirror-view';
-import React, { memo, useEffect, useMemo, useRef, useState } from 'react';
+import React, { memo, useEffect, useRef } from 'react';
 import { renderToString } from 'react-dom/server';
 
 import { Suggestion } from '@/db/schema';

lib/editor/DiffType.js

@@ -1,7 +0,0 @@
-// Taken from https://github.com/hamflx/prosemirror-diff/blob/master/src/DiffType.js
-
-export const DiffType = {
-  Unchanged: 0,
-  Deleted: -1,
-  Inserted: 1,
-};

lib/editor/diff.js

@@ -1,9 +1,13 @@
-// Taken from https://github.com/hamflx/prosemirror-diff/blob/master/src/diff.js
+// Modified from https://github.com/hamflx/prosemirror-diff/blob/master/src/diff.js
 
 import { diff_match_patch } from 'diff-match-patch';
 import { Fragment, Node } from 'prosemirror-model';
 
-import { DiffType } from './DiffType.js';
+export const DiffType = {
+  Unchanged: 0,
+  Deleted: -1,
+  Inserted: 1,
+};
 
 export const patchDocumentNode = (schema, oldNode, newNode) => {
   assertNodeTypeEqual(oldNode, newNode);
@@ -20,7 +24,6 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
   let left = 0;
   let right = 0;
 
-  // console.log("==> searching same left");
   for (; left < minChildLen; left++) {
     const oldChild = oldChildren[left];
     const newChild = newChildren[left];
@@ -30,7 +33,6 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
     finalLeftChildren.push(...ensureArray(oldChild));
   }
 
-  // console.log("==> searching same right");
   for (; right + left + 1 < minChildLen; right++) {
     const oldChild = oldChildren[oldChildLen - right - 1];
     const newChild = newChildren[newChildLen - right - 1];
@@ -40,19 +42,9 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
     finalRightChildren.unshift(...ensureArray(oldChild));
   }
 
-  // console.log(
-  //   `==> eq left:${left}, right:${right}`,
-  //   [...finalLeftChildren],
-  //   [...finalRightChildren],
-  // );
-
   const diffOldChildren = oldChildren.slice(left, oldChildLen - right);
   const diffNewChildren = newChildren.slice(left, newChildLen - right);
-  // console.log(
+
-  //   "==> diff children",
-  //   diffOldChildren.length,
-  //   diffNewChildren.length,
-  // );
   if (diffOldChildren.length && diffNewChildren.length) {
     const matchedNodes = matchNodes(
       schema,
@@ -61,18 +53,11 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
     ).sort((a, b) => b.count - a.count);
     const bestMatch = matchedNodes[0];
     if (bestMatch) {
-      // console.log("==> bestMatch", bestMatch);
       const { oldStartIndex, newStartIndex, oldEndIndex, newEndIndex } =
         bestMatch;
       const oldBeforeMatchChildren = diffOldChildren.slice(0, oldStartIndex);
       const newBeforeMatchChildren = diffNewChildren.slice(0, newStartIndex);
-      // console.log(
+
-      //   "==> before match",
-      //   oldBeforeMatchChildren.length,
-      //   newBeforeMatchChildren.length,
-      //   oldBeforeMatchChildren,
-      //   newBeforeMatchChildren,
-      // );
       finalLeftChildren.push(
         ...patchRemainNodes(
           schema,
@@ -83,15 +68,10 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
       finalLeftChildren.push(
         ...diffOldChildren.slice(oldStartIndex, oldEndIndex)
       );
-      // console.log("==> match", oldEndIndex - oldStartIndex);
 
       const oldAfterMatchChildren = diffOldChildren.slice(oldEndIndex);
       const newAfterMatchChildren = diffNewChildren.slice(newEndIndex);
-      // console.log(
+
-      //   "==> after match",
-      //   oldAfterMatchChildren.length,
-      //   newAfterMatchChildren.length,
-      // );
       finalRightChildren.unshift(
         ...patchRemainNodes(
           schema,
@@ -100,12 +80,10 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
         )
       );
     } else {
-      // console.log("==> no best match found");
       finalLeftChildren.push(
         ...patchRemainNodes(schema, diffOldChildren, diffNewChildren)
       );
     }
-    // console.log("==> matchedNodes", matchedNodes);
   } else {
     finalLeftChildren.push(
       ...patchRemainNodes(schema, diffOldChildren, diffNewChildren)
@@ -116,7 +94,6 @@ export const patchDocumentNode = (schema, oldNode, newNode) => {
 };
 
 const matchNodes = (schema, oldChildren, newChildren) => {
-  // console.log("==> matchNodes", oldChildren, newChildren);
   const matches = [];
   for (
     let oldStartIndex = 0;
@@ -139,13 +116,6 @@ const matchNodes = (schema, oldChildren, newChildren) => {
           break;
         }
       }
-      // console.log(
-      //   "==> match",
-      //   oldStartIndex,
-      //   oldEndIndex,
-      //   newStartIndex,
-      //   newEndIndex,
-      // );
       matches.push({
         oldStartIndex,
         newStartIndex,
@@ -214,7 +184,7 @@ const patchRemainNodes = (schema, oldChildren, newChildren) => {
       );
       right += 1;
     } else {
-      // todo
+      // Delete and insert
       finalLeftChildren.push(
         createDiffNode(schema, leftOldNode, DiffType.Deleted)
       );
@@ -222,7 +192,6 @@ const patchRemainNodes = (schema, oldChildren, newChildren) => {
         createDiffNode(schema, leftNewNode, DiffType.Inserted)
       );
       left += 1;
-      // delete and insert
     }
   }
 
@@ -249,104 +218,92 @@ const patchRemainNodes = (schema, oldChildren, newChildren) => {
   return [...finalLeftChildren, ...finalRightChildren];
 };
 
+// Updated function to perform sentence-level diffs
 export const patchTextNodes = (schema, oldNode, newNode) => {
   const dmp = new diff_match_patch();
+
+  // Concatenate the text from the text nodes
   const oldText = oldNode.map((n) => getNodeText(n)).join('');
   const newText = newNode.map((n) => getNodeText(n)).join('');
-  const diff = dmp.diff_main(oldText, newText);
-  let oldLen = 0;
-  let newLen = 0;
-  const res = diff
-    .map((d) => {
-      const [type, content] = [d[0], d[1]];
-      const node = createTextNode(
-        schema,
-        content,
-        type !== DiffType.Unchanged ? createDiffMark(schema, type) : []
-      );
-      const oldFrom = oldLen;
-      const oldTo = oldFrom + (type === DiffType.Inserted ? 0 : content.length);
-      const newFrom = newLen;
-      const newTo = newFrom + (type === DiffType.Deleted ? 0 : content.length);
-      oldLen = oldTo;
-      newLen = newTo;
-      return { node, type, oldFrom, oldTo, newFrom, newTo };
-    })
-    .map(({ node, type, oldFrom, oldTo, newFrom, newTo }) => {
-      if (type === DiffType.Deleted) {
-        const textItems = findTextNodes(oldNode, oldFrom, oldTo).filter(
-          (n) => Object.keys(n.node.attrs ?? {}).length || n.node.marks?.length
-        );
-        return applyTextNodeAttrsMarks(schema, node, oldFrom, textItems);
-      } else {
-        const textItems = findTextNodes(newNode, newFrom, newTo).filter(
-          (n) => Object.keys(n.node.attrs ?? {}).length || n.node.marks?.length
-        );
-        return applyTextNodeAttrsMarks(schema, node, newFrom, textItems);
-      }
-    });
-  return res.flat(Infinity);
-};
 
-const findTextNodes = (textNodes, from, to) => {
+  // Tokenize the text into sentences
-  const result = [];
+  const oldSentences = tokenizeSentences(oldText);
-  let start = 0;
+  const newSentences = tokenizeSentences(newText);
-  for (let i = 0; i < textNodes.length && start < to; i++) {
-    const node = textNodes[i];
-    const text = getNodeText(node);
-    const end = start + text.length;
-    const intersect =
-      (start >= from && start < to) ||
-      (end > from && end <= to) ||
-      (start <= from && end >= to);
-    if (intersect) {
-      result.push({ node, from: start, to: end });
-    }
-    start += text.length;
-  }
-  return result;
-};
 
-const applyTextNodeAttrsMarks = (schema, node, base, textItems) => {
+  // Map sentences to unique characters
-  if (!textItems.length) {
+  const { chars1, chars2, lineArray } = sentencesToChars(
-    return node;
+    oldSentences,
-  }
+    newSentences
+  );
 
-  const baseMarks = node.marks ?? [];
+  // Perform the diff
-  const firstItem = textItems[0];
+  let diffs = dmp.diff_main(chars1, chars2, false);
-  const nodeText = getNodeText(node);
+
-  const nodeEnd = base + nodeText.length;
+  // Convert back to sentences
-  const result = [];
+  diffs = diffs.map(([type, text]) => {
-  if (firstItem.from - base > 0) {
+    const sentences = text
-    result.push(
+      .split('')
-      createTextNode(
+      .map((char) => lineArray[char.charCodeAt(0)]);
-        schema,
+    return [type, sentences];
-        nodeText.slice(0, firstItem.from - base),
+  });
-        baseMarks
+
-      )
+  // Map diffs to nodes
-    );
+  const res = diffs
-  }
+    .map(([type, sentences]) => {
-  for (let i = 0; i < textItems.length; i++) {
+      return sentences.map((sentence) => {
-    const { from, node: textNode, to } = textItems[i];
+        const node = createTextNode(
-    result.push(
-      createTextNode(
-        schema,
-        nodeText.slice(Math.max(from, base) - base, to - base),
-        [...baseMarks, ...(textNode.marks ?? [])]
-      )
-    );
-    const nextFrom = i + 1 < textItems.length ? textItems[i + 1].from : nodeEnd;
-    if (nextFrom > to) {
-      result.push(
-        createTextNode(
           schema,
-          nodeText.slice(to - base, nextFrom - base),
+          sentence,
-          baseMarks
+          type !== DiffType.Unchanged ? [createDiffMark(schema, type)] : []
-        )
+        );
-      );
+        return node;
-    }
+      });
-  }
+    })
-  return result;
+    .flat();
+
+  return res;
+};
+
+// Function to tokenize text into sentences
+const tokenizeSentences = (text) => {
+  return text.match(/[^.!?]+[.!?]*\s*/g) || [];
+};
+
+// Function to map sentences to unique characters
+const sentencesToChars = (oldSentences, newSentences) => {
+  const lineArray = [];
+  const lineHash = {};
+  let lineStart = 0;
+
+  const chars1 = oldSentences
+    .map((sentence) => {
+      const line = sentence;
+      if (lineHash.hasOwnProperty(line)) {
+        return String.fromCharCode(lineHash[line]);
+      } else {
+        lineHash[line] = lineStart;
+        lineArray[lineStart] = line;
+        lineStart++;
+        return String.fromCharCode(lineHash[line]);
+      }
+    })
+    .join('');
+
+  const chars2 = newSentences
+    .map((sentence) => {
+      const line = sentence;
+      if (lineHash.hasOwnProperty(line)) {
+        return String.fromCharCode(lineHash[line]);
+      } else {
+        lineHash[line] = lineStart;
+        lineArray[lineStart] = line;
+        lineStart++;
+        return String.fromCharCode(lineHash[line]);
+      }
+    })
+    .join('');
+
+  return { chars1, chars2, lineArray };
 };
 
 export const computeChildEqualityFactor = (node1, node2) => {
@@ -448,17 +405,24 @@ export const getNodeProperty = (node, property) => {
   }
   return node[property];
 };
+
 export const getNodeAttribute = (node, attribute) =>
   node.attrs ? node.attrs[attribute] : undefined;
-export const getNodeAttributes = (node) =>
+
-  node.attrs ? node.attrs : undefined;
+export const getNodeAttributes = (node) => (node.attrs ? node.attrs : {});
+
 export const getNodeMarks = (node) => node.marks ?? [];
+
 export const getNodeChildren = (node) => node.content?.content ?? [];
+
 export const getNodeText = (node) => node.text;
+
 export const isTextNode = (node) => node.type?.name === 'text';
+
 export const matchNodeType = (node1, node2) =>
   node1.type?.name === node2.type?.name ||
   (Array.isArray(node1) && Array.isArray(node2));
+
 export const createNewNode = (oldNode, children) => {
   if (!oldNode.type) {
     throw new Error('oldNode.type is undefined');
@@ -470,6 +434,7 @@ export const createNewNode = (oldNode, children) => {
     oldNode.marks
   );
 };
+
 export const createDiffNode = (schema, node, type) => {
   return mapDocumentNode(node, (node) => {
     if (isTextNode(node)) {
@@ -481,6 +446,7 @@ export const createDiffNode = (schema, node, type) => {
     return node;
   });
 };
+
 function mapDocumentNode(node, mapper) {
   const copy = node.copy(
     Fragment.from(
@@ -491,6 +457,7 @@ function mapDocumentNode(node, mapper) {
   );
   return mapper(copy) || copy;
 }
+
 export const createDiffMark = (schema, type) => {
   if (type === DiffType.Inserted) {
     return schema.mark('diffMark', { type });
@@ -500,6 +467,7 @@ export const createDiffMark = (schema, type) => {
   }
   throw new Error('type is not valid');
 };
+
 export const createTextNode = (schema, content, marks = []) => {
   return schema.text(content, marks);
 };

lib/editor/index.js

@@ -1,2 +0,0 @@
-export * from "./diff.js";
-export * from "./DiffType";

lib/editor/react-renderer.tsx

@@ -1,9 +1,10 @@
-import { createRoot } from "react-dom/client";
+import { createRoot } from 'react-dom/client';
 
 export class ReactRenderer {
   static render(component: React.ReactElement, dom: HTMLElement) {
     const root = createRoot(dom);
     root.render(component);
+
     return {
       destroy: () => root.unmount(),
     };