Enhance the number parser to include support for parsing hexadecimal,…

… binary, and octal literals. (#483) * Enhance the number parser to include support for parsing hexadecimal, binary, and octal literals. * added invalid literal test cases * Refactoring code as per comment --------- Co-authored-by: Ganesan Karuppasamy <[email protected]>
expr-lang · Dec 8, 2023 · 87c6a94 · 87c6a94
1 parent 128b621
commit 87c6a94
Show file tree

Hide file tree

Showing 3 changed files with 103 additions and 21 deletions.
diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go
@@ -18,14 +18,16 @@ func TestLex(t *testing.T) {
 		tokens []Token
 	}{
 		{
-			".5 0.025 1 02 1e3 0xFF 1.2e-4 1_000_000 _42 -.5",
+			".5 0.025 1 02 1e3 0xFF 0b0101 0o600 1.2e-4 1_000_000 _42 -.5",
 			[]Token{
 				{Kind: Number, Value: ".5"},
 				{Kind: Number, Value: "0.025"},
 				{Kind: Number, Value: "1"},
 				{Kind: Number, Value: "02"},
 				{Kind: Number, Value: "1e3"},
 				{Kind: Number, Value: "0xFF"},
+				{Kind: Number, Value: "0b0101"},
+				{Kind: Number, Value: "0o600"},
 				{Kind: Number, Value: "1.2e-4"},
 				{Kind: Number, Value: "1_000_000"},
 				{Kind: Identifier, Value: "_42"},

diff --git a/parser/parser.go b/parser/parser.go
@@ -306,40 +306,44 @@ func (p *parser) parseSecondary() Node {
 	case Number:
 		p.next()
 		value := strings.Replace(token.Value, "_", "", -1)
-		if strings.Contains(value, "x") {
+		var node Node
+		valueLower := strings.ToLower(value)
+		switch {
+		case strings.HasPrefix(valueLower, "0x"):
 			number, err := strconv.ParseInt(value, 0, 64)
 			if err != nil {
 				p.error("invalid hex literal: %v", err)
 			}
-			if number > math.MaxInt {
-				p.error("integer literal is too large")
-				return nil
-			}
-			node := &IntegerNode{Value: int(number)}
-			node.SetLocation(token.Location)
-			return node
-		} else if strings.ContainsAny(value, ".eE") {
+			node = p.toIntegerNode(number)
+		case strings.ContainsAny(valueLower, ".e"):
 			number, err := strconv.ParseFloat(value, 64)
 			if err != nil {
 				p.error("invalid float literal: %v", err)
 			}
-			node := &FloatNode{Value: number}
-			node.SetLocation(token.Location)
-			return node
-		} else {
+			node = p.toFloatNode(number)
+		case strings.HasPrefix(valueLower, "0b"):
+			number, err := strconv.ParseInt(value, 0, 64)
+			if err != nil {
+				p.error("invalid binary literal: %v", err)
+			}
+			node = p.toIntegerNode(number)
+		case strings.HasPrefix(valueLower, "0o"):
+			number, err := strconv.ParseInt(value, 0, 64)
+			if err != nil {
+				p.error("invalid octal literal: %v", err)
+			}
+			node = p.toIntegerNode(number)
+		default:
 			number, err := strconv.ParseInt(value, 10, 64)
 			if err != nil {
 				p.error("invalid integer literal: %v", err)
 			}
-			if number > math.MaxInt {
-				p.error("integer literal is too large")
-				return nil
-			}
-			node := &IntegerNode{Value: int(number)}
+			node = p.toIntegerNode(number)
+		}
+		if node != nil {
 			node.SetLocation(token.Location)
-			return node
 		}
-
+		return node
 	case String:
 		p.next()
 		node := &StringNode{Value: token.Value}
@@ -359,6 +363,22 @@ func (p *parser) parseSecondary() Node {
 	return p.parsePostfixExpression(node)
 }
 
+func (p *parser) toIntegerNode(number int64) Node {
+	if number > math.MaxInt {
+		p.error("integer literal is too large")
+		return nil
+	}
+	return &IntegerNode{Value: int(number)}
+}
+
+func (p *parser) toFloatNode(number float64) Node {
+	if number > math.MaxFloat64 {
+		p.error("float literal is too large")
+		return nil
+	}
+	return &FloatNode{Value: number}
+}
+
 func (p *parser) parseCall(token Token) Node {
 	var node Node
 	if p.current.Is(Bracket, "(") {

diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -36,6 +36,26 @@ func TestParse(t *testing.T) {
 			"0x6E",
 			&IntegerNode{Value: 110},
 		},
+		{
+			"0X63",
+			&IntegerNode{Value: 99},
+		},
+		{
+			"0o600",
+			&IntegerNode{Value: 384},
+		},
+		{
+			"0O45",
+			&IntegerNode{Value: 37},
+		},
+		{
+			"0b10",
+			&IntegerNode{Value: 2},
+		},
+		{
+			"0B101011",
+			&IntegerNode{Value: 43},
+		},
 		{
 			"10_000_000",
 			&IntegerNode{Value: 10_000_000},
@@ -549,6 +569,46 @@ foo ?? bar || baz
 Operator (||) and coalesce expressions (??) cannot be mixed. Wrap either by parentheses. (1:12)
  | foo ?? bar || baz
  | ...........^
+
+0b15
+bad number syntax: "0b15" (1:5)
+ | 0b15
+ | ....^
+
+0X10G
+bad number syntax: "0X10G" (1:6)
+ | 0X10G
+ | .....^
+
+0o1E
+invalid float literal: strconv.ParseFloat: parsing "0o1E": invalid syntax (1:4)
+ | 0o1E
+ | ...^
+
+0b1E
+invalid float literal: strconv.ParseFloat: parsing "0b1E": invalid syntax (1:4)
+ | 0b1E
+ | ...^
+
+0b1E+6
+bad number syntax: "0b1E+6" (1:7)
+ | 0b1E+6
+ | ......^
+
+0b1E+1
+invalid float literal: strconv.ParseFloat: parsing "0b1E+1": invalid syntax (1:6)
+ | 0b1E+1
+ | .....^
+
+0o1E+1
+invalid float literal: strconv.ParseFloat: parsing "0o1E+1": invalid syntax (1:6)
+ | 0o1E+1
+ | .....^
+
+1E
+invalid float literal: strconv.ParseFloat: parsing "1E": invalid syntax (1:2)
+ | 1E
+ | .^
 `
 
 func TestParse_error(t *testing.T) {