editor: Fix wide characters

- Display correctly.
- Fix cursor movement.
- Work correctly with tabs.

diff_edit/editor.py

@@ -16,6 +16,8 @@ import pygments.lexers
 import pygments.styles
 import termstr
 
+import cwcwidth
+
 
 @functools.lru_cache(maxsize=100)
 def highlight_str(line, bg_color, transparency=0.6):
@@ -59,6 +61,12 @@ def _syntax_highlight(text, lexer, style):
     return fill3.join("\n", text_widget.text)
 
 
+@functools.lru_cache(maxsize=500)
+def expand_str(str_):
+    expanded_str = termstr.TermStr(str_)
+    return str_ if expanded_str.data == str_ else expanded_str
+
+
 class Text:
 
     def __init__(self, text, padding_char=" "):
@@ -85,7 +93,7 @@ class Text:
             self._replace_lines(key, value)
 
     def _replace_lines(self, slice_, new_lines):
-        fixed_lines = [line.expandtabs() for line in new_lines]
+        fixed_lines = [expand_str(line) for line in new_lines]
         max_new_lengths = max(len(line) for line in fixed_lines)
         if max_new_lengths > self.max_line_length:
             padding = self.padding_char * (max_new_lengths - self.max_line_length)
@@ -93,7 +101,7 @@ class Text:
             self.max_line_length = max_new_lengths
         converted_lines = [self._convert_line(line, self.max_line_length) for line in fixed_lines]
         self.text[slice_], self.actual_text[slice_] = converted_lines, new_lines
-        new_max_line_length = max(len(line.expandtabs()) for line in self.actual_text)
+        new_max_line_length = max(len(expand_str(line)) for line in self.actual_text)
         if new_max_line_length < self.max_line_length:
             clip_width = self.max_line_length - new_max_line_length
             self.text = [line[:-clip_width] for line in self.text]
@@ -155,16 +163,12 @@ def highlight_part(line, start, end):
             line[end:])
 
 
-@functools.lru_cache(maxsize=100)
+@functools.lru_cache(maxsize=500)
-def expandtabs_inverse(s):
+def expand_str_inverse(str_):
-    parts_len = [len(part) for part in s.split("\t")]
+    result = []
-    result = list(range(parts_len[0]))
+    for index, char in enumerate(str_):
-    cursor = parts_len[0]
+        run_length = 8 - len(result) % 8 if char == "\t" else cwcwidth.wcwidth(char)
-    for part_len in parts_len[1:]:
+        result.extend([index] * run_length)
-        result.extend([cursor] * (8 - (len(result) % 8)))
-        cursor += 1
-        result.extend(range(cursor, cursor + part_len))
-        cursor += part_len
     return result
 
 
@@ -190,12 +194,17 @@ class Editor:
 
     @property
     def cursor_x(self):
-        line_length = len(self.text_widget.actual_text[self.cursor_y])
+        try:
-        return min(self._cursor_x, line_length)
+            return expand_str_inverse(self.text_widget[self.cursor_y])[self._cursor_x]
+        except IndexError:
+            return len(self.text_widget.actual_text[self.cursor_y])
 
     @cursor_x.setter
     def cursor_x(self, x):
-        self._cursor_x = x
+        try:
+            self._cursor_x = len(expand_str(self.text_widget[self.cursor_y][:x]))
+        except IndexError:
+            self._cursor_x = x
 
     @property
     def cursor_y(self):
@@ -229,8 +238,8 @@ class Editor:
             result[self.cursor_y] = highlight_str(result[self.cursor_y], termstr.Color.white, 0.8)
         else:
             (start_x, start_y), (end_x, end_y) = self.get_selection_interval()
-            screen_start_x = len(self.text_widget[start_y][:start_x].expandtabs())
+            screen_start_x = len(expand_str(self.text_widget[start_y][:start_x]))
-            screen_end_x = len(self.text_widget[end_y][:end_x].expandtabs())
+            screen_end_x = len(expand_str(self.text_widget[end_y][:end_x]))
             if start_y == end_y:
                 result[start_y] = highlight_part(result[start_y], screen_start_x, screen_end_x)
             else:
@@ -242,10 +251,12 @@ class Editor:
         if self.cursor_x >= len(result[0]):
             result = fill3.appearance_resize(result, (self.cursor_x+1, len(result)))
         cursor_line = result[self.cursor_y]
-        screen_x = len(self.text_widget[self.cursor_y][:self.cursor_x].expandtabs())
+        screen_x = len(expand_str(self.text_widget[self.cursor_y][:self.cursor_x]))
+        screen_x_after = (screen_x + 1 if self._current_character() in ["\t", "\n"]
+                          else len(expand_str(self.text_widget[self.cursor_y][:self.cursor_x+1])))
         result[self.cursor_y] = (cursor_line[:screen_x] +
-                                 termstr.TermStr(cursor_line[screen_x]).invert() +
+                                 termstr.TermStr(cursor_line[screen_x:screen_x_after]).invert() +
-                                 cursor_line[screen_x+1:])
+                                 cursor_line[screen_x_after:])
         return result
 
     def set_text(self, text):
@@ -259,7 +270,7 @@ class Editor:
         self.view_widget.portal.is_scroll_limited = True
         if not self.is_left_aligned:
             self.view_widget.portal.is_left_aligned = False
-        self.cursor_x, self.cursor_y = 0, 0
+        self._cursor_x, self._cursor_y = 0, 0
         self.original_text = self.text_widget.actual_text.copy()
 
     def load(self, path):
@@ -482,7 +493,7 @@ class Editor:
                 new_line = (self.text_widget[start_y][:start_x] + "# " +
                             self.text_widget[start_y][start_x:])
                 self.text_widget[start_y] = new_line
-                self.cursor_x = len(new_line)
+                self._cursor_x = len(new_line)
                 start_y += 1
             if end_x != 0:
                 end_y += 1
@@ -599,7 +610,7 @@ class Editor:
             new_y = self.cursor_y - height // 2
         else:
             new_y = view_y
-        screen_x = len(self.text_widget[self.cursor_y][:self.cursor_x].expandtabs())
+        screen_x = len(expand_str(self.text_widget[self.cursor_y][:self.cursor_x]))
         if screen_x >= view_x + width or screen_x < view_x:
             new_x = screen_x - width // 2
         else:
@@ -633,10 +644,7 @@ class Editor:
     def on_mouse_press(self, x, y):
         view_x, view_y = self.view_widget.position
         self.cursor_y = min(y + view_y - 1, len(self.text_widget) - 1)
-        try:
+        self._cursor_x = x + view_x
-            self.cursor_x = expandtabs_inverse(self.text_widget[self.cursor_y])[x + view_x]
-        except IndexError:
-            self.cursor_x = expandtabs_inverse(self.text_widget[self.cursor_y])[-1]
         self.last_mouse_position = (x, y)
 
     def on_mouse_drag(self, x, y):

setup.py

@@ -18,4 +18,4 @@ setup(name="diff-edit",
       entry_points={"console_scripts": ["diff-edit=diff_edit:main"]},
       install_requires=[
           "pygments==2.10.0", "docopt==0.6.2",
-          "fill3 @ git+https://github.com/ahamilton/eris@v2022.02.13#subdirectory=fill3"])
+          "fill3 @ git+https://github.com/ahamilton/eris@v2022.02.23#subdirectory=fill3"])

tests/editor_test.py

@@ -3,6 +3,8 @@
 
 import unittest
 
+import termstr
+
 import diff_edit.editor as editor
 
 
@@ -50,18 +52,21 @@ class TextWidgetTestCase(unittest.TestCase):
     def test_tabs(self):
         text = editor.Text("a\tb\naa\tb")
         self.assertEqual(text.get_text(), "a\tb\naa\tb")
-        self.assertEqual(text.appearance(), ["a       b", "aa      b"])
+        self.assertEqual(text.appearance(),
+                         [termstr.TermStr("a       b"), termstr.TermStr("aa      b")])
         text = editor.Text("a\tb\tc")
-        self.assertEqual(text.appearance(), ["a       b       c"])
+        self.assertEqual(text.appearance(), [termstr.TermStr("a       b       c")])
 
-    def test_expandtabs_inverse(self):
+    def test_expand_str_inverse(self):
-        self.assertEqual(editor.expandtabs_inverse(""), [])
+        self.assertEqual(editor.expand_str_inverse(""), [])
-        self.assertEqual(editor.expandtabs_inverse("a"), [0])
+        self.assertEqual(editor.expand_str_inverse("a"), [0])
-        self.assertEqual(editor.expandtabs_inverse("a\tb"), [0, 1, 1, 1, 1, 1, 1, 1, 2])
+        self.assertEqual(editor.expand_str_inverse("a\tb"), [0, 1, 1, 1, 1, 1, 1, 1, 2])
-        self.assertEqual(editor.expandtabs_inverse("aaaaaaaaaa\t"),
+        self.assertEqual(editor.expand_str_inverse("aaaaaaaaaa\t"),
                          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 10])
-        self.assertEqual(editor.expandtabs_inverse("a\tb\tc"),
+        self.assertEqual(editor.expand_str_inverse("a\tb\tc"),
                          [0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 4])
+        self.assertEqual(editor.expand_str_inverse("♓"), [0, 0])
+        self.assertEqual(editor.expand_str_inverse("♓\tb"), [0, 0, 1, 1, 1, 1, 1, 1, 2])
 
 
 class EditorTestCase(unittest.TestCase):
@@ -77,7 +82,7 @@ class EditorTestCase(unittest.TestCase):
 
     def _set_editor(self, text, cursor_position):
         self.editor.set_text(text)
-        self.editor.cursor_x, self.editor.cursor_y = cursor_position
+        self.editor._cursor_x, self.editor._cursor_y = cursor_position
 
     def _assert_change(self, method, expected_text, expected_cursor_position):
         method()
@@ -183,6 +188,11 @@ class EditorTestCase(unittest.TestCase):
         self._assert_change(self.editor.cursor_down, text, (2, 1))
         self._assert_change(self.editor.jump_to_start_of_line, text, (0, 1))
         self._assert_change(self.editor.jump_to_end_of_line, text, (2, 1))
+        text = ("♓\n"
+                "bc")
+        self._set_editor(text, (0, 0))
+        self._assert_change(self.editor.cursor_right, text, (1, 0))
+        self._assert_change(self.editor.cursor_down, text, (2, 1))
 
     def test_jumping_words(self):
         text = ("ab .dj\n"