diff --git a/src/lexer/numbers.js b/src/lexer/numbers.js index 484b35a99..edb5e4022 100644 --- a/src/lexer/numbers.js +++ b/src/lexer/numbers.js @@ -23,7 +23,7 @@ module.exports = { if (ch === "x" || ch === "X") { ch = this.input(); if (ch !== "_" && this.is_HEX()) { - return this.consume_HNUM(); + return this.consume_hexadecimal(); } else { this.unput(ch ? 2 : 1); } @@ -31,14 +31,14 @@ module.exports = { } else if (ch === "b" || ch === "B") { ch = this.input(); if ((ch !== "_" && ch === "0") || ch === "1") { - return this.consume_BNUM(); + return this.consume_binary(); } else { this.unput(ch ? 2 : 1); } } else if (ch === "o" || ch === "O") { ch = this.input(); if (ch !== "_" && this.is_OCTAL()) { - return this.consume_ONUM(); + return this.consume_octal(); } else { this.unput(ch ? 2 : 1); } @@ -94,7 +94,7 @@ module.exports = { ch = this.input(); } if (this.is_NUM_START()) { - this.consume_LNUM(); + this.consume_decimal_digits(); return this.tok.T_DNUMBER; } this.unput(ch ? undo : undo - 1); // keep only 1 @@ -123,49 +123,44 @@ module.exports = { return this.tok.T_DNUMBER; } }, - // read hexa - consume_HNUM() { + consume_prefixed_digits(isValid) { + let prev = this._input[this.offset - 1]; while (this.offset < this.size) { const ch = this.input(); - if (!this.is_HEX()) { + if (!isValid.call(this)) { if (ch) this.unput(1); break; } - } - return this.tok.T_LNUMBER; - }, - // read a generic number - consume_LNUM() { - while (this.offset < this.size) { - const ch = this.input(); - if (!this.is_NUM()) { - if (ch) this.unput(1); + if (ch === "_" && prev === "_") { + this.unput(2); + prev = null; break; } + prev = ch; } + if (prev === "_") this.unput(1); return this.tok.T_LNUMBER; }, - // read binary - consume_BNUM() { - let ch; - while (this.offset < this.size) { - ch = this.input(); - if (ch !== "0" && ch !== "1" && ch !== "_") { - if (ch) this.unput(1); - break; - } - } - return this.tok.T_LNUMBER; + consume_hexadecimal() { + return this.consume_prefixed_digits(this.is_HEX); }, - // read an octal number - consume_ONUM() { + consume_decimal_digits() { while (this.offset < this.size) { const ch = this.input(); - if (!this.is_OCTAL()) { + if (!this.is_NUM()) { if (ch) this.unput(1); break; } } return this.tok.T_LNUMBER; }, + consume_binary() { + return this.consume_prefixed_digits(function () { + const ch = this._input[this.offset - 1]; + return ch === "0" || ch === "1" || ch === "_"; + }); + }, + consume_octal() { + return this.consume_prefixed_digits(this.is_OCTAL); + }, }; diff --git a/test/snapshot/__snapshots__/number.test.js.snap b/test/snapshot/__snapshots__/number.test.js.snap index a9bfd38de..d7a2f1a77 100644 --- a/test/snapshot/__snapshots__/number.test.js.snap +++ b/test/snapshot/__snapshots__/number.test.js.snap @@ -1,5 +1,98 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing +exports[`Test numbers binary consecutive underscores 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "e", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0b1", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": undefined, + "kind": "expressionstatement", + }, + ], + "errors": [ + Error { + "expected": ";", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error, unexpected '__0' (T_STRING), expecting ';' on line 1", + "token": "'__0' (T_STRING)", + }, + Error { + "expected": "EXPR", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error on line 1", + "token": "the end of file (EOF)", + }, + ], + "kind": "program", +} +`; + +exports[`Test numbers binary trailing underscore 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "e", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0b1", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": Bin { + "kind": "bin", + "left": Name { + "kind": "name", + "name": "_", + "resolution": "uqn", + }, + "right": Number { + "kind": "number", + "value": "1", + }, + "type": "+", + }, + "kind": "expressionstatement", + }, + ], + "errors": [ + Error { + "expected": ";", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error, unexpected '_' (T_STRING), expecting ';' on line 1", + "token": "'_' (T_STRING)", + }, + ], + "kind": "program", +} +`; + exports[`Test numbers binary with 2 1`] = ` Program { "children": [ @@ -138,6 +231,99 @@ Program { } `; +exports[`Test numbers hex consecutive underscores 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "e", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0x1", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": undefined, + "kind": "expressionstatement", + }, + ], + "errors": [ + Error { + "expected": ";", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error, unexpected '__2' (T_STRING), expecting ';' on line 1", + "token": "'__2' (T_STRING)", + }, + Error { + "expected": "EXPR", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error on line 1", + "token": "the end of file (EOF)", + }, + ], + "kind": "program", +} +`; + +exports[`Test numbers hex trailing underscore 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "e", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0x1", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": Bin { + "kind": "bin", + "left": Name { + "kind": "name", + "name": "_", + "resolution": "uqn", + }, + "right": Number { + "kind": "number", + "value": "1", + }, + "type": "+", + }, + "kind": "expressionstatement", + }, + ], + "errors": [ + Error { + "expected": ";", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error, unexpected '_' (T_STRING), expecting ';' on line 1", + "token": "'_' (T_STRING)", + }, + ], + "kind": "program", +} +`; + exports[`Test numbers hexa without hex 1`] = ` Program { "children": [ @@ -222,6 +408,99 @@ Program { } `; +exports[`Test numbers octal consecutive underscores 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "e", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0o7", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": undefined, + "kind": "expressionstatement", + }, + ], + "errors": [ + Error { + "expected": ";", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error, unexpected '__1' (T_STRING), expecting ';' on line 1", + "token": "'__1' (T_STRING)", + }, + Error { + "expected": "EXPR", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error on line 1", + "token": "the end of file (EOF)", + }, + ], + "kind": "program", +} +`; + +exports[`Test numbers octal trailing underscore 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "e", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0o7", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": Bin { + "kind": "bin", + "left": Name { + "kind": "name", + "name": "_", + "resolution": "uqn", + }, + "right": Number { + "kind": "number", + "value": "1", + }, + "type": "+", + }, + "kind": "expressionstatement", + }, + ], + "errors": [ + Error { + "expected": ";", + "kind": "error", + "line": 1, + "message": "Parse Error : syntax error, unexpected '_' (T_STRING), expecting ';' on line 1", + "token": "'_' (T_STRING)", + }, + ], + "kind": "program", +} +`; + exports[`Test numbers test common cases 1`] = ` Program { "children": [ @@ -402,6 +681,63 @@ Program { } `; +exports[`Test numbers test underscore separators in hex/binary/octal 1`] = ` +Program { + "children": [ + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "a", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0x1_A2_B3", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "b", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0b1010_0101", + }, + }, + "kind": "expressionstatement", + }, + ExpressionStatement { + "expression": Assign { + "kind": "assign", + "left": Variable { + "curly": false, + "kind": "variable", + "name": "c", + }, + "operator": "=", + "right": Number { + "kind": "number", + "value": "0o7_6_5", + }, + }, + "kind": "expressionstatement", + }, + ], + "errors": [], + "kind": "program", +} +`; + exports[`Test numbers underscore #1 1`] = ` Program { "children": [ diff --git a/test/snapshot/number.test.js b/test/snapshot/number.test.js index 84b6960b0..c018e76fd 100644 --- a/test/snapshot/number.test.js +++ b/test/snapshot/number.test.js @@ -19,6 +19,16 @@ describe("Test numbers", function () { ).toMatchSnapshot(); }); + it("test underscore separators in hex/binary/octal", function () { + expect( + parser.parseEval(` + $a = 0x1_A2_B3; + $b = 0b1010_0101; + $c = 0o7_6_5; + `), + ).toMatchSnapshot(); + }); + it.each([ ["hexa without hex", "$a = 0xx;"], ["binary with 2", "$b = 0b2;"], @@ -33,6 +43,12 @@ describe("Test numbers", function () { ["underscore #3", "$e = 7_.0;"], ["underscore #4", "$e = 7e_0;"], ["underscore #5", "$e = 7_e0;"], + ["hex consecutive underscores", "$e = 0x1__2;"], + ["hex trailing underscore", "$e = 0x1_ + 1;"], + ["binary consecutive underscores", "$e = 0b1__0;"], + ["binary trailing underscore", "$e = 0b1_ + 1;"], + ["octal consecutive underscores", "$e = 0o7__1;"], + ["octal trailing underscore", "$e = 0o7_ + 1;"], ])("%s", function (_, code) { const ast = parser.parseEval(code, { parser: {