From d303c5b3ebc6d56af6d3a03c6b4cdb361a2b022c Mon Sep 17 00:00:00 2001
From: Junegunn Choi <junegunn.c@gmail.com>
Date: Fri, 9 Jan 2015 02:35:20 +0900
Subject: Minor refactoring

---
 src/tokenizer.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'src/tokenizer.go')

diff --git a/src/tokenizer.go b/src/tokenizer.go
index bc1ca3a4..d62f395d 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -118,14 +118,13 @@ func awkTokenizer(input *string) ([]string, int) {
 }
 
 func Tokenize(str *string, delimiter *regexp.Regexp) []Token {
-	prefixLength := 0
 	if delimiter == nil {
 		// AWK-style (\S+\s*)
 		tokens, prefixLength := awkTokenizer(str)
 		return withPrefixLengths(tokens, prefixLength)
 	} else {
 		tokens := delimiter.FindAllString(*str, -1)
-		return withPrefixLengths(tokens, prefixLength)
+		return withPrefixLengths(tokens, 0)
 	}
 }
 
-- 
cgit v1.2.3