From d9c8a9a880dd9fc28c9fb466bd1c81cd6a8a159c Mon Sep 17 00:00:00 2001
From: Junegunn Choi <junegunn.c@gmail.com>
Date: Sun, 14 Aug 2016 04:23:37 +0900
Subject: [perf] Remove memory copy when using string delimiter

---
 src/tokenizer.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'src/tokenizer.go')

diff --git a/src/tokenizer.go b/src/tokenizer.go
index eec19898..e48f48bf 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -140,13 +140,13 @@ func Tokenize(text util.Chars, delimiter Delimiter) []Token {
 		return withPrefixLengths(tokens, prefixLength)
 	}
 
-	var tokens []string
 	if delimiter.str != nil {
-		tokens = strings.Split(text.ToString(), *delimiter.str)
-		for i := 0; i < len(tokens)-1; i++ {
-			tokens[i] = tokens[i] + *delimiter.str
-		}
-	} else if delimiter.regex != nil {
+		return withPrefixLengths(text.Split(*delimiter.str), 0)
+	}
+
+	// FIXME performance
+	var tokens []string
+	if delimiter.regex != nil {
 		str := text.ToString()
 		for len(str) > 0 {
 			loc := delimiter.regex.FindStringIndex(str)
-- 
cgit v1.2.3