[perf] Optimize AWK-style tokenizer for --nth

Approx. 50% less memory footprint and 40% improvement in query time

1 files changed, 21 insertions, 0 deletions

diff --git a/src/util/chars_test.go b/src/util/chars_test.go
index e42cfb7c..2cb6fc76 100644
--- a/src/util/chars_test.go
+++ b/src/util/chars_test.go
@@ -34,3 +34,24 @@ func TestCharsToString(t *testing.T) {
 		t.Error()
 	}
 }
+
+func TestTrimLength(t *testing.T) {
+	check := func(str string, exp int) {
+		chars := ToChars([]byte(str))
+		trimmed := chars.TrimLength()
+		if trimmed != exp {
+			t.Errorf("Invalid TrimLength result for '%s': %d (expected %d)",
+				str, trimmed, exp)
+		}
+	}
+	check("hello", 5)
+	check("hello ", 5)
+	check("hello  ", 5)
+	check(" hello", 5)
+	check("  hello", 5)
+	check(" hello ", 5)
+	check("  hello  ", 5)
+	check("h   o", 5)
+	check("  h   o  ", 5)
+	check("         ", 0)
+}