From f64d582a4ecfe74529db53c88c8e7695f78860ab Mon Sep 17 00:00:00 2001
From: Liam Fitzgerald <liam.fitzgerald@tlon.io>
Date: Thu, 22 Apr 2021 18:53:42 +1000
Subject: [PATCH] tokeniseMessage: handle linebreaks correctly

---
 .../src/logic/lib/tokenizeMessage.js          | 52 +++++++++++--------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/pkg/interface/src/logic/lib/tokenizeMessage.js b/pkg/interface/src/logic/lib/tokenizeMessage.js
index 0e9c12d59..fecb2d8f6 100644
--- a/pkg/interface/src/logic/lib/tokenizeMessage.js
+++ b/pkg/interface/src/logic/lib/tokenizeMessage.js
@@ -17,15 +17,15 @@ const isRef = (str) => {
 
 const tokenizeMessage = (text) => {
   let messages = [];
-  let message = [];
+  // by line
+  let currTextBlock = [];
   let isInCodeBlock = false;
   let endOfCodeBlock = false;
   text.split(/\r?\n/).forEach((line, index) => {
-    if (index !== 0) {
-      message.push('\n');
-    }
+    // by space
+    let currTextLine = [];
     // A line of backticks enters and exits a codeblock
-    if (line.startsWith('```')) {
+    if (line.trim().startsWith('```')) {
       // But we need to check if we've ended a codeblock
       endOfCodeBlock = isInCodeBlock;
       isInCodeBlock = (!isInCodeBlock);
@@ -34,9 +34,11 @@ const tokenizeMessage = (text) => {
     }
 
     if (isInCodeBlock || endOfCodeBlock) {
-      message.push(line);
+      currTextLine = [line];
     } else {
-      line.split(/\s/).forEach((str) => {
+      const words = line.split(/\s/);
+      words.forEach((str, idx) => {
+        const last = words.length - 1 === idx;
         if (
           (str.startsWith('`') && str !== '`')
           || (str === '`' && !isInCodeBlock)
@@ -50,9 +52,12 @@ const tokenizeMessage = (text) => {
         }
 
         if(isRef(str) && !isInCodeBlock) {
-          if (message.length > 0) {
+          if (currTextLine.length > 0 || currTextBlock.length > 0) {
             // If we're in the middle of a message, add it to the stack and reset
-            messages.push({ text: message.join(' ') });
+            currTextLine.push('');
+            messages.push({ text: currTextBlock.join('\n') + currTextLine.join(' ') });
+            currTextBlock = last ? [''] : [];
+            currTextLine = [];
           }
           const link = parsePermalink(str);
           if(!link) {
@@ -61,34 +66,39 @@ const tokenizeMessage = (text) => {
             const reference = permalinkToReference(link);
             messages.push(reference);
           }
-          message = [];
+          currTextLine = [];
         } else if (isUrl(str) && !isInCodeBlock) {
-          if (message.length > 0) {
+          if (currTextLine.length > 0 || currTextBlock.length > 0) {
             // If we're in the middle of a message, add it to the stack and reset
-            messages.push({ text: message.join(' ') });
-            message = [];
+            currTextLine.push('');
+            messages.push({ text: currTextBlock.join('\n') + currTextLine.join(' ') });
+            currTextBlock = last ? [''] : [];
+            currTextLine = [];
           }
           messages.push({ url: str });
-          message = [];
+          currTextLine = [];
         } else if(urbitOb.isValidPatp(str) && !isInCodeBlock) {
-          if (message.length > 0) {
+          if (currTextLine.length > 0 || currTextBlock.length > 0) {
             // If we're in the middle of a message, add it to the stack and reset
-            messages.push({ text: message.join(' ') });
-            message = [];
+            currTextLine.push('');
+            messages.push({ text: currTextBlock.join('\n') + currTextLine.join(' ') });
+            currTextBlock = last ? [''] : [];
+            currTextLine = [];
           }
           messages.push({ mention: str });
-          message = [];
+          currTextLine = [];
 
         } else {
-          message.push(str);
+          currTextLine.push(str);
         }
       });
     }
+    currTextBlock.push(currTextLine.join(' '))
   });
 
-  if (message.length) {
+  if (currTextBlock.length) {
     // Add any remaining message
-    messages.push({ text: message.join(' ') });
+    messages.push({ text: currTextBlock.join('\n') });
   }
   return messages;
 };