From af809c966150f4623f9316e8b63e146491134e50 Mon Sep 17 00:00:00 2001
From: Junegunn Choi <junegunn.c@gmail.com>
Date: Sat, 26 Aug 2017 03:24:42 +0900
Subject: Minor refactorings

---
 src/tokenizer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/tokenizer.go')

diff --git a/src/tokenizer.go b/src/tokenizer.go
index 5b7a8b6f..6c1d8cab 100644
--- a/src/tokenizer.go
+++ b/src/tokenizer.go
@@ -147,7 +147,7 @@ func Tokenize(text string, delimiter Delimiter) []Token {
 	if delimiter.regex != nil {
 		for len(text) > 0 {
 			loc := delimiter.regex.FindStringIndex(text)
-			if loc == nil {
+			if len(loc) < 2 {
 				loc = []int{0, len(text)}
 			}
 			last := util.Max(loc[1], 1)
-- 
cgit v1.2.3