Merge pull request #50 from yaacov/review-round2-fixes

fixes

Author: yaacov

v6/pkg/parser/ast.go

@@ -146,7 +146,9 @@ func (op OpType) String() string {
 
 // Node represents a generic AST node
 type Node struct {
-	Kind     NodeKind
+	Kind NodeKind
+	// Value must be an immutable type (string, float64, bool, time.Time, or nil).
+	// Clone performs a shallow copy of this field.
 	Value    interface{}
 	Operator OpType
 	Left     *Node

v6/pkg/parser/bench_test.go

@@ -0,0 +1,37 @@
+package parser
+
+import "testing"
+
+func BenchmarkParseSimple(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("name = 'alice'"); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkParseComplex(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		if _, err := Parse("(age > 25 and city = 'rome') or (status in ['active', 'pending'] and price between 10 and 100)"); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkLexerSimple(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		l := NewLexer("name = 'alice'")
+		if err := l.Tokenize(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkLexerComplex(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		l := NewLexer("(age > 25 and city = 'rome') or (status in ['active', 'pending'] and price between 10 and 100)")
+		if err := l.Tokenize(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

v6/pkg/parser/fuzz_test.go

@@ -0,0 +1,25 @@
+package parser
+
+import "testing"
+
+func FuzzParse(f *testing.F) {
+	f.Add("name = 'alice'")
+	f.Add("age > 25 and city = 'rome'")
+	f.Add("status in ['active', 'pending']")
+	f.Add("price between 10 and 100")
+	f.Add("not (deleted = true)")
+	f.Add("title like '%book%'")
+	f.Add("created_at > 2023-01-01")
+	f.Add("(a = 1 or b = 2) and c = 3")
+	f.Add("")
+	f.Add("   ")
+	f.Add("名前 = 'test'")
+	f.Add("name = '🎉'")
+	f.Add("[1, 2, 3]")
+	f.Add("a ~= '.*'")
+
+	f.Fuzz(func(t *testing.T, input string) {
+		// Should not panic regardless of input
+		_, _ = Parse(input)
+	})
+}

v6/pkg/parser/lexer.go

@@ -21,9 +21,10 @@ const (
 
 // Lexer represents a lexical analyzer for TSL
 type Lexer struct {
-	input   string
-	pos     int // current position
-	start   int // start of current token
+	input   string // original input string
+	runes   []rune // input as runes for proper UTF-8 handling
+	pos     int    // current position (rune index)
+	start   int    // start of current token (rune index)
 	tokens  []Token
 	current int // current token index
 }
@@ -58,6 +59,7 @@ var (
 func NewLexer(input string) *Lexer {
 	return &Lexer{
 		input:   input,
+		runes:   []rune(input),
 		pos:     0,
 		start:   0,
 		tokens:  make([]Token, 0),
@@ -80,38 +82,38 @@ func (l *Lexer) Tokenize() error {
 
 // isAtEnd checks if we're at the end of input
 func (l *Lexer) isAtEnd() bool {
-	return l.pos >= len(l.input)
+	return l.pos >= len(l.runes)
 }
 
 // peek returns the current character without advancing
 func (l *Lexer) peek() rune {
 	if l.isAtEnd() {
 		return 0
 	}
-	return rune(l.input[l.pos])
+	return l.runes[l.pos]
 }
 
 // peekNext returns the next character without advancing
 func (l *Lexer) peekNext() rune {
-	if l.pos+1 >= len(l.input) {
+	if l.pos+1 >= len(l.runes) {
 		return 0
 	}
-	return rune(l.input[l.pos+1])
+	return l.runes[l.pos+1]
 }
 
 // advance consumes and returns the current character
 func (l *Lexer) advance() rune {
 	if l.isAtEnd() {
 		return 0
 	}
-	c := rune(l.input[l.pos])
+	c := l.runes[l.pos]
 	l.pos++
 	return c
 }
 
 // match checks if current character matches expected and advances if so
 func (l *Lexer) match(expected rune) bool {
-	if l.isAtEnd() || rune(l.input[l.pos]) != expected {
+	if l.isAtEnd() || l.runes[l.pos] != expected {
 		return false
 	}
 	l.pos++
@@ -227,11 +229,11 @@ func (l *Lexer) scanToken() error {
 // isDateTimePattern checks if the current position starts a date or time pattern
 func (l *Lexer) isDateTimePattern() bool {
 	// Look ahead to see if this matches a date or RFC3339 pattern
-	remaining := l.input[l.pos:]
+	remaining := l.runes[l.pos:]
 
 	// Try to match date pattern (YYYY-MM-DD)
 	if len(remaining) >= 10 {
-		dateCandidate := remaining[:10]
+		dateCandidate := string(remaining[:10])
 		if datePattern.MatchString(dateCandidate) {
 			// Check if it continues as RFC3339 (has T after date)
 			if len(remaining) > 10 && remaining[10] == 'T' {
@@ -240,12 +242,12 @@ func (l *Lexer) isDateTimePattern() bool {
 					c := remaining[i]
 					if c == ' ' || c == '\t' || c == '\n' || c == ')' || c == ',' {
 						// End of potential timestamp
-						candidate := remaining[:i]
+						candidate := string(remaining[:i])
 						return rfc3339Pattern.MatchString(candidate)
 					}
 				}
 				// Check the whole remaining string
-				return rfc3339Pattern.MatchString(remaining)
+				return rfc3339Pattern.MatchString(string(remaining))
 			}
 			return true // Just a date
 		}
@@ -267,7 +269,7 @@ func (l *Lexer) scanDateTime() error {
 		l.advance()
 	}
 
-	value := l.input[start:l.pos]
+	value := string(l.runes[start:l.pos])
 
 	// Check if it's a date or RFC3339 format
 	if rfc3339Pattern.MatchString(value) {
@@ -391,7 +393,7 @@ func (l *Lexer) scanNumber() error {
 		}
 	}
 
-	value := l.input[start:l.pos]
+	value := string(l.runes[start:l.pos])
 	l.addToken(NUMERIC_LITERAL, value)
 	return nil
 }
@@ -430,7 +432,7 @@ func (l *Lexer) scanIdentifier() error {
 		}
 	}
 
-	value := l.input[start:l.pos]
+	value := string(l.runes[start:l.pos])
 
 	// Check if it's a keyword (case-insensitive)
 	lowerValue := strings.ToLower(value)
@@ -446,7 +448,7 @@ func (l *Lexer) scanIdentifier() error {
 // NextToken returns the next token for the parser
 func (l *Lexer) NextToken() Token {
 	if l.current >= len(l.tokens) {
-		return Token{Type: EOF, Value: "", Position: len(l.input)}
+		return Token{Type: EOF, Value: "", Position: len(l.runes)}
 	}
 	token := l.tokens[l.current]
 	l.current++
@@ -456,7 +458,7 @@ func (l *Lexer) NextToken() Token {
 // Peek returns the current token without advancing
 func (l *Lexer) PeekToken() Token {
 	if l.current >= len(l.tokens) {
-		return Token{Type: EOF, Value: "", Position: len(l.input)}
+		return Token{Type: EOF, Value: "", Position: len(l.runes)}
 	}
 	return l.tokens[l.current]
 }