From 0d06c28b1943e820440cff9182a0ae3bdd5e3c59 Mon Sep 17 00:00:00 2001
From: Junegunn Choi <junegunn.c@gmail.com>
Date: Thu, 21 Jul 2022 21:21:06 +0900
Subject: Fix delimiter regex to properly support caret (^)

Fix #2861
---
 src/tokenizer.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'src/tokenizer.go')

diff --git a/src/tokenizer.go b/src/tokenizer.go
index 26f42d25..9f9e2c19 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -156,14 +156,14 @@ func Tokenize(text string, delimiter Delimiter) []Token {
 	// FIXME performance
 	var tokens []string
 	if delimiter.regex != nil {
-		for len(text) > 0 {
-			loc := delimiter.regex.FindStringIndex(text)
-			if len(loc) < 2 {
-				loc = []int{0, len(text)}
-			}
-			last := util.Max(loc[1], 1)
-			tokens = append(tokens, text[:last])
-			text = text[last:]
+		locs := delimiter.regex.FindAllStringIndex(text, -1)
+		begin := 0
+		for _, loc := range locs {
+			tokens = append(tokens, text[begin:loc[1]])
+			begin = loc[1]
+		}
+		if begin < len(text) {
+			tokens = append(tokens, text[begin:])
 		}
 	}
 	return withPrefixLengths(tokens, 0)
-- 
cgit v1.2.3