diff options
Diffstat (limited to 'node_modules/babylon/lib/tokenizer')
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/context.js | 99 | ||||
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/index.js | 889 | ||||
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/state.js | 146 | ||||
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/types.js | 159 |
4 files changed, 1293 insertions, 0 deletions
diff --git a/node_modules/babylon/lib/tokenizer/context.js b/node_modules/babylon/lib/tokenizer/context.js new file mode 100644 index 000000000..856051a01 --- /dev/null +++ b/node_modules/babylon/lib/tokenizer/context.js @@ -0,0 +1,99 @@ +"use strict"; + +exports.__esModule = true; +exports.types = exports.TokContext = undefined; + +var _types = require("./types"); + +var _whitespace = require("../util/whitespace"); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } // The algorithm used to determine whether a regexp can appear at a +// given point in the program is loosely based on sweet.js' approach. +// See https://github.com/mozilla/sweet.js/wiki/design + +var TokContext = exports.TokContext = function TokContext(token, isExpr, preserveSpace, override) { + _classCallCheck(this, TokContext); + + this.token = token; + this.isExpr = !!isExpr; + this.preserveSpace = !!preserveSpace; + this.override = override; +}; + +var types = exports.types = { + braceStatement: new TokContext("{", false), + braceExpression: new TokContext("{", true), + templateQuasi: new TokContext("${", true), + parenStatement: new TokContext("(", false), + parenExpression: new TokContext("(", true), + template: new TokContext("`", true, true, function (p) { + return p.readTmplToken(); + }), + functionExpression: new TokContext("function", true) +}; + +// Token-specific context update code + +_types.types.parenR.updateContext = _types.types.braceR.updateContext = function () { + if (this.state.context.length === 1) { + this.state.exprAllowed = true; + return; + } + + var out = this.state.context.pop(); + if (out === types.braceStatement && this.curContext() === types.functionExpression) { + this.state.context.pop(); + this.state.exprAllowed = false; + } else if (out === types.templateQuasi) { + this.state.exprAllowed = true; + } else { + this.state.exprAllowed = !out.isExpr; + } +}; + +_types.types.name.updateContext = function (prevType) { + this.state.exprAllowed = false; + + if (prevType === _types.types._let || prevType === _types.types._const || prevType === _types.types._var) { + if (_whitespace.lineBreak.test(this.input.slice(this.state.end))) { + this.state.exprAllowed = true; + } + } +}; + +_types.types.braceL.updateContext = function (prevType) { + this.state.context.push(this.braceIsBlock(prevType) ? types.braceStatement : types.braceExpression); + this.state.exprAllowed = true; +}; + +_types.types.dollarBraceL.updateContext = function () { + this.state.context.push(types.templateQuasi); + this.state.exprAllowed = true; +}; + +_types.types.parenL.updateContext = function (prevType) { + var statementParens = prevType === _types.types._if || prevType === _types.types._for || prevType === _types.types._with || prevType === _types.types._while; + this.state.context.push(statementParens ? types.parenStatement : types.parenExpression); + this.state.exprAllowed = true; +}; + +_types.types.incDec.updateContext = function () { + // tokExprAllowed stays unchanged +}; + +_types.types._function.updateContext = function () { + if (this.curContext() !== types.braceStatement) { + this.state.context.push(types.functionExpression); + } + + this.state.exprAllowed = false; +}; + +_types.types.backQuote.updateContext = function () { + if (this.curContext() === types.template) { + this.state.context.pop(); + } else { + this.state.context.push(types.template); + } + this.state.exprAllowed = false; +};
\ No newline at end of file diff --git a/node_modules/babylon/lib/tokenizer/index.js b/node_modules/babylon/lib/tokenizer/index.js new file mode 100644 index 000000000..25d2b8d13 --- /dev/null +++ b/node_modules/babylon/lib/tokenizer/index.js @@ -0,0 +1,889 @@ +"use strict"; + +exports.__esModule = true; +exports.Token = undefined; + +var _identifier = require("../util/identifier"); + +var _types = require("./types"); + +var _context = require("./context"); + +var _location = require("../util/location"); + +var _whitespace = require("../util/whitespace"); + +var _state = require("./state"); + +var _state2 = _interopRequireDefault(_state); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } /* eslint max-len: 0 */ +/* eslint indent: 0 */ + +// Object type used to represent tokens. Note that normally, tokens +// simply exist as properties on the parser object. This is only +// used for the onToken callback and the external tokenizer. + +var Token = exports.Token = function Token(state) { + _classCallCheck(this, Token); + + this.type = state.type; + this.value = state.value; + this.start = state.start; + this.end = state.end; + this.loc = new _location.SourceLocation(state.startLoc, state.endLoc); +}; + +// ## Tokenizer + +function codePointToString(code) { + // UTF-16 Decoding + if (code <= 0xFFFF) { + return String.fromCharCode(code); + } else { + return String.fromCharCode((code - 0x10000 >> 10) + 0xD800, (code - 0x10000 & 1023) + 0xDC00); + } +} + +var Tokenizer = function () { + function Tokenizer(options, input) { + _classCallCheck(this, Tokenizer); + + this.state = new _state2.default(); + this.state.init(options, input); + } + + // Move to the next token + + Tokenizer.prototype.next = function next() { + if (!this.isLookahead) { + this.state.tokens.push(new Token(this.state)); + } + + this.state.lastTokEnd = this.state.end; + this.state.lastTokStart = this.state.start; + this.state.lastTokEndLoc = this.state.endLoc; + this.state.lastTokStartLoc = this.state.startLoc; + this.nextToken(); + }; + + // TODO + + Tokenizer.prototype.eat = function eat(type) { + if (this.match(type)) { + this.next(); + return true; + } else { + return false; + } + }; + + // TODO + + Tokenizer.prototype.match = function match(type) { + return this.state.type === type; + }; + + // TODO + + Tokenizer.prototype.isKeyword = function isKeyword(word) { + return (0, _identifier.isKeyword)(word); + }; + + // TODO + + Tokenizer.prototype.lookahead = function lookahead() { + var old = this.state; + this.state = old.clone(true); + + this.isLookahead = true; + this.next(); + this.isLookahead = false; + + var curr = this.state.clone(true); + this.state = old; + return curr; + }; + + // Toggle strict mode. Re-reads the next number or string to please + // pedantic tests (`"use strict"; 010;` should fail). + + Tokenizer.prototype.setStrict = function setStrict(strict) { + this.state.strict = strict; + if (!this.match(_types.types.num) && !this.match(_types.types.string)) return; + this.state.pos = this.state.start; + while (this.state.pos < this.state.lineStart) { + this.state.lineStart = this.input.lastIndexOf("\n", this.state.lineStart - 2) + 1; + --this.state.curLine; + } + this.nextToken(); + }; + + Tokenizer.prototype.curContext = function curContext() { + return this.state.context[this.state.context.length - 1]; + }; + + // Read a single token, updating the parser object's token-related + // properties. + + Tokenizer.prototype.nextToken = function nextToken() { + var curContext = this.curContext(); + if (!curContext || !curContext.preserveSpace) this.skipSpace(); + + this.state.containsOctal = false; + this.state.octalPosition = null; + this.state.start = this.state.pos; + this.state.startLoc = this.state.curPosition(); + if (this.state.pos >= this.input.length) return this.finishToken(_types.types.eof); + + if (curContext.override) { + return curContext.override(this); + } else { + return this.readToken(this.fullCharCodeAtPos()); + } + }; + + Tokenizer.prototype.readToken = function readToken(code) { + // Identifier or keyword. '\uXXXX' sequences are allowed in + // identifiers, so '\' also dispatches to that. + if ((0, _identifier.isIdentifierStart)(code) || code === 92 /* '\' */) { + return this.readWord(); + } else { + return this.getTokenFromCode(code); + } + }; + + Tokenizer.prototype.fullCharCodeAtPos = function fullCharCodeAtPos() { + var code = this.input.charCodeAt(this.state.pos); + if (code <= 0xd7ff || code >= 0xe000) return code; + + var next = this.input.charCodeAt(this.state.pos + 1); + return (code << 10) + next - 0x35fdc00; + }; + + Tokenizer.prototype.pushComment = function pushComment(block, text, start, end, startLoc, endLoc) { + var comment = { + type: block ? "CommentBlock" : "CommentLine", + value: text, + start: start, + end: end, + loc: new _location.SourceLocation(startLoc, endLoc) + }; + + if (!this.isLookahead) { + this.state.tokens.push(comment); + this.state.comments.push(comment); + this.addComment(comment); + } + }; + + Tokenizer.prototype.skipBlockComment = function skipBlockComment() { + var startLoc = this.state.curPosition(); + var start = this.state.pos, + end = this.input.indexOf("*/", this.state.pos += 2); + if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment"); + + this.state.pos = end + 2; + _whitespace.lineBreakG.lastIndex = start; + var match = void 0; + while ((match = _whitespace.lineBreakG.exec(this.input)) && match.index < this.state.pos) { + ++this.state.curLine; + this.state.lineStart = match.index + match[0].length; + } + + this.pushComment(true, this.input.slice(start + 2, end), start, this.state.pos, startLoc, this.state.curPosition()); + }; + + Tokenizer.prototype.skipLineComment = function skipLineComment(startSkip) { + var start = this.state.pos; + var startLoc = this.state.curPosition(); + var ch = this.input.charCodeAt(this.state.pos += startSkip); + while (this.state.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) { + ++this.state.pos; + ch = this.input.charCodeAt(this.state.pos); + } + + this.pushComment(false, this.input.slice(start + startSkip, this.state.pos), start, this.state.pos, startLoc, this.state.curPosition()); + }; + + // Called at the start of the parse and after every token. Skips + // whitespace and comments, and. + + Tokenizer.prototype.skipSpace = function skipSpace() { + loop: while (this.state.pos < this.input.length) { + var ch = this.input.charCodeAt(this.state.pos); + switch (ch) { + case 32:case 160: + // ' ' + ++this.state.pos; + break; + + case 13: + if (this.input.charCodeAt(this.state.pos + 1) === 10) { + ++this.state.pos; + } + + case 10:case 8232:case 8233: + ++this.state.pos; + ++this.state.curLine; + this.state.lineStart = this.state.pos; + break; + + case 47: + // '/' + switch (this.input.charCodeAt(this.state.pos + 1)) { + case 42: + // '*' + this.skipBlockComment(); + break; + + case 47: + this.skipLineComment(2); + break; + + default: + break loop; + } + break; + + default: + if (ch > 8 && ch < 14 || ch >= 5760 && _whitespace.nonASCIIwhitespace.test(String.fromCharCode(ch))) { + ++this.state.pos; + } else { + break loop; + } + } + } + }; + + // Called at the end of every token. Sets `end`, `val`, and + // maintains `context` and `exprAllowed`, and skips the space after + // the token, so that the next one's `start` will point at the + // right position. + + Tokenizer.prototype.finishToken = function finishToken(type, val) { + this.state.end = this.state.pos; + this.state.endLoc = this.state.curPosition(); + var prevType = this.state.type; + this.state.type = type; + this.state.value = val; + + this.updateContext(prevType); + }; + + // ### Token reading + + // This is the function that is called to fetch the next token. It + // is somewhat obscure, because it works in character codes rather + // than characters, and because operator parsing has been inlined + // into it. + // + // All in the name of speed. + // + + + Tokenizer.prototype.readToken_dot = function readToken_dot() { + var next = this.input.charCodeAt(this.state.pos + 1); + if (next >= 48 && next <= 57) { + return this.readNumber(true); + } + + var next2 = this.input.charCodeAt(this.state.pos + 2); + if (next === 46 && next2 === 46) { + // 46 = dot '.' + this.state.pos += 3; + return this.finishToken(_types.types.ellipsis); + } else { + ++this.state.pos; + return this.finishToken(_types.types.dot); + } + }; + + Tokenizer.prototype.readToken_slash = function readToken_slash() { + // '/' + if (this.state.exprAllowed) { + ++this.state.pos; + return this.readRegexp(); + } + + var next = this.input.charCodeAt(this.state.pos + 1); + if (next === 61) { + return this.finishOp(_types.types.assign, 2); + } else { + return this.finishOp(_types.types.slash, 1); + } + }; + + Tokenizer.prototype.readToken_mult_modulo = function readToken_mult_modulo(code) { + // '%*' + var type = code === 42 ? _types.types.star : _types.types.modulo; + var width = 1; + var next = this.input.charCodeAt(this.state.pos + 1); + + if (next === 42) { + // '*' + width++; + next = this.input.charCodeAt(this.state.pos + 2); + type = _types.types.exponent; + } + + if (next === 61) { + width++; + type = _types.types.assign; + } + + return this.finishOp(type, width); + }; + + Tokenizer.prototype.readToken_pipe_amp = function readToken_pipe_amp(code) { + // '|&' + var next = this.input.charCodeAt(this.state.pos + 1); + if (next === code) return this.finishOp(code === 124 ? _types.types.logicalOR : _types.types.logicalAND, 2); + if (next === 61) return this.finishOp(_types.types.assign, 2); + if (code === 124 && next === 125 && this.hasPlugin("flow")) return this.finishOp(_types.types.braceBarR, 2); + return this.finishOp(code === 124 ? _types.types.bitwiseOR : _types.types.bitwiseAND, 1); + }; + + Tokenizer.prototype.readToken_caret = function readToken_caret() { + // '^' + var next = this.input.charCodeAt(this.state.pos + 1); + if (next === 61) { + return this.finishOp(_types.types.assign, 2); + } else { + return this.finishOp(_types.types.bitwiseXOR, 1); + } + }; + + Tokenizer.prototype.readToken_plus_min = function readToken_plus_min(code) { + // '+-' + var next = this.input.charCodeAt(this.state.pos + 1); + + if (next === code) { + if (next === 45 && this.input.charCodeAt(this.state.pos + 2) === 62 && _whitespace.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))) { + // A `-->` line comment + this.skipLineComment(3); + this.skipSpace(); + return this.nextToken(); + } + return this.finishOp(_types.types.incDec, 2); + } + + if (next === 61) { + return this.finishOp(_types.types.assign, 2); + } else { + return this.finishOp(_types.types.plusMin, 1); + } + }; + + Tokenizer.prototype.readToken_lt_gt = function readToken_lt_gt(code) { + // '<>' + var next = this.input.charCodeAt(this.state.pos + 1); + var size = 1; + + if (next === code) { + size = code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2; + if (this.input.charCodeAt(this.state.pos + size) === 61) return this.finishOp(_types.types.assign, size + 1); + return this.finishOp(_types.types.bitShift, size); + } + + if (next === 33 && code === 60 && this.input.charCodeAt(this.state.pos + 2) === 45 && this.input.charCodeAt(this.state.pos + 3) === 45) { + if (this.inModule) this.unexpected(); + // `<!--`, an XML-style comment that should be interpreted as a line comment + this.skipLineComment(4); + this.skipSpace(); + return this.nextToken(); + } + + if (next === 61) { + // <= | >= + size = 2; + } + + return this.finishOp(_types.types.relational, size); + }; + + Tokenizer.prototype.readToken_eq_excl = function readToken_eq_excl(code) { + // '=!' + var next = this.input.charCodeAt(this.state.pos + 1); + if (next === 61) return this.finishOp(_types.types.equality, this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2); + if (code === 61 && next === 62) { + // '=>' + this.state.pos += 2; + return this.finishToken(_types.types.arrow); + } + return this.finishOp(code === 61 ? _types.types.eq : _types.types.prefix, 1); + }; + + Tokenizer.prototype.getTokenFromCode = function getTokenFromCode(code) { + switch (code) { + // The interpretation of a dot depends on whether it is followed + // by a digit or another two dots. + case 46: + // '.' + return this.readToken_dot(); + + // Punctuation tokens. + case 40: + ++this.state.pos;return this.finishToken(_types.types.parenL); + case 41: + ++this.state.pos;return this.finishToken(_types.types.parenR); + case 59: + ++this.state.pos;return this.finishToken(_types.types.semi); + case 44: + ++this.state.pos;return this.finishToken(_types.types.comma); + case 91: + ++this.state.pos;return this.finishToken(_types.types.bracketL); + case 93: + ++this.state.pos;return this.finishToken(_types.types.bracketR); + + case 123: + if (this.hasPlugin("flow") && this.input.charCodeAt(this.state.pos + 1) === 124) { + return this.finishOp(_types.types.braceBarL, 2); + } else { + ++this.state.pos; + return this.finishToken(_types.types.braceL); + } + + case 125: + ++this.state.pos;return this.finishToken(_types.types.braceR); + + case 58: + if (this.hasPlugin("functionBind") && this.input.charCodeAt(this.state.pos + 1) === 58) { + return this.finishOp(_types.types.doubleColon, 2); + } else { + ++this.state.pos; + return this.finishToken(_types.types.colon); + } + + case 63: + ++this.state.pos;return this.finishToken(_types.types.question); + case 64: + ++this.state.pos;return this.finishToken(_types.types.at); + + case 96: + // '`' + ++this.state.pos; + return this.finishToken(_types.types.backQuote); + + case 48: + // '0' + var next = this.input.charCodeAt(this.state.pos + 1); + if (next === 120 || next === 88) return this.readRadixNumber(16); // '0x', '0X' - hex number + if (next === 111 || next === 79) return this.readRadixNumber(8); // '0o', '0O' - octal number + if (next === 98 || next === 66) return this.readRadixNumber(2); // '0b', '0B' - binary number + // Anything else beginning with a digit is an integer, octal + // number, or float. + case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57: + // 1-9 + return this.readNumber(false); + + // Quotes produce strings. + case 34:case 39: + // '"', "'" + return this.readString(code); + + // Operators are parsed inline in tiny state machines. '=' (61) is + // often referred to. `finishOp` simply skips the amount of + // characters it is given as second argument, and returns a token + // of the type given by its first argument. + + case 47: + // '/' + return this.readToken_slash(); + + case 37:case 42: + // '%*' + return this.readToken_mult_modulo(code); + + case 124:case 38: + // '|&' + return this.readToken_pipe_amp(code); + + case 94: + // '^' + return this.readToken_caret(); + + case 43:case 45: + // '+-' + return this.readToken_plus_min(code); + + case 60:case 62: + // '<>' + return this.readToken_lt_gt(code); + + case 61:case 33: + // '=!' + return this.readToken_eq_excl(code); + + case 126: + // '~' + return this.finishOp(_types.types.prefix, 1); + } + + this.raise(this.state.pos, "Unexpected character '" + codePointToString(code) + "'"); + }; + + Tokenizer.prototype.finishOp = function finishOp(type, size) { + var str = this.input.slice(this.state.pos, this.state.pos + size); + this.state.pos += size; + return this.finishToken(type, str); + }; + + Tokenizer.prototype.readRegexp = function readRegexp() { + var escaped = void 0, + inClass = void 0, + start = this.state.pos; + for (;;) { + if (this.state.pos >= this.input.length) this.raise(start, "Unterminated regular expression"); + var ch = this.input.charAt(this.state.pos); + if (_whitespace.lineBreak.test(ch)) { + this.raise(start, "Unterminated regular expression"); + } + if (escaped) { + escaped = false; + } else { + if (ch === "[") { + inClass = true; + } else if (ch === "]" && inClass) { + inClass = false; + } else if (ch === "/" && !inClass) { + break; + } + escaped = ch === "\\"; + } + ++this.state.pos; + } + var content = this.input.slice(start, this.state.pos); + ++this.state.pos; + // Need to use `readWord1` because '\uXXXX' sequences are allowed + // here (don't ask). + var mods = this.readWord1(); + if (mods) { + var validFlags = /^[gmsiyu]*$/; + if (!validFlags.test(mods)) this.raise(start, "Invalid regular expression flag"); + } + return this.finishToken(_types.types.regexp, { + pattern: content, + flags: mods + }); + }; + + // Read an integer in the given radix. Return null if zero digits + // were read, the integer value otherwise. When `len` is given, this + // will return `null` unless the integer has exactly `len` digits. + + Tokenizer.prototype.readInt = function readInt(radix, len) { + var start = this.state.pos, + total = 0; + for (var i = 0, e = len == null ? Infinity : len; i < e; ++i) { + var code = this.input.charCodeAt(this.state.pos), + val = void 0; + if (code >= 97) { + val = code - 97 + 10; // a + } else if (code >= 65) { + val = code - 65 + 10; // A + } else if (code >= 48 && code <= 57) { + val = code - 48; // 0-9 + } else { + val = Infinity; + } + if (val >= radix) break; + ++this.state.pos; + total = total * radix + val; + } + if (this.state.pos === start || len != null && this.state.pos - start !== len) return null; + + return total; + }; + + Tokenizer.prototype.readRadixNumber = function readRadixNumber(radix) { + this.state.pos += 2; // 0x + var val = this.readInt(radix); + if (val == null) this.raise(this.state.start + 2, "Expected number in radix " + radix); + if ((0, _identifier.isIdentifierStart)(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number"); + return this.finishToken(_types.types.num, val); + }; + + // Read an integer, octal integer, or floating-point number. + + Tokenizer.prototype.readNumber = function readNumber(startsWithDot) { + var start = this.state.pos, + isFloat = false, + octal = this.input.charCodeAt(this.state.pos) === 48; + if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number"); + var next = this.input.charCodeAt(this.state.pos); + if (next === 46) { + // '.' + ++this.state.pos; + this.readInt(10); + isFloat = true; + next = this.input.charCodeAt(this.state.pos); + } + if (next === 69 || next === 101) { + // 'eE' + next = this.input.charCodeAt(++this.state.pos); + if (next === 43 || next === 45) ++this.state.pos; // '+-' + if (this.readInt(10) === null) this.raise(start, "Invalid number"); + isFloat = true; + } + if ((0, _identifier.isIdentifierStart)(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number"); + + var str = this.input.slice(start, this.state.pos), + val = void 0; + if (isFloat) { + val = parseFloat(str); + } else if (!octal || str.length === 1) { + val = parseInt(str, 10); + } else if (/[89]/.test(str) || this.state.strict) { + this.raise(start, "Invalid number"); + } else { + val = parseInt(str, 8); + } + return this.finishToken(_types.types.num, val); + }; + + // Read a string value, interpreting backslash-escapes. + + Tokenizer.prototype.readCodePoint = function readCodePoint() { + var ch = this.input.charCodeAt(this.state.pos), + code = void 0; + + if (ch === 123) { + var codePos = ++this.state.pos; + code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos); + ++this.state.pos; + if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds"); + } else { + code = this.readHexChar(4); + } + return code; + }; + + Tokenizer.prototype.readString = function readString(quote) { + var out = "", + chunkStart = ++this.state.pos; + for (;;) { + if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated string constant"); + var ch = this.input.charCodeAt(this.state.pos); + if (ch === quote) break; + if (ch === 92) { + // '\' + out += this.input.slice(chunkStart, this.state.pos); + out += this.readEscapedChar(false); + chunkStart = this.state.pos; + } else { + if ((0, _whitespace.isNewLine)(ch)) this.raise(this.state.start, "Unterminated string constant"); + ++this.state.pos; + } + } + out += this.input.slice(chunkStart, this.state.pos++); + return this.finishToken(_types.types.string, out); + }; + + // Reads template string tokens. + + Tokenizer.prototype.readTmplToken = function readTmplToken() { + var out = "", + chunkStart = this.state.pos; + for (;;) { + if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated template"); + var ch = this.input.charCodeAt(this.state.pos); + if (ch === 96 || ch === 36 && this.input.charCodeAt(this.state.pos + 1) === 123) { + // '`', '${' + if (this.state.pos === this.state.start && this.match(_types.types.template)) { + if (ch === 36) { + this.state.pos += 2; + return this.finishToken(_types.types.dollarBraceL); + } else { + ++this.state.pos; + return this.finishToken(_types.types.backQuote); + } + } + out += this.input.slice(chunkStart, this.state.pos); + return this.finishToken(_types.types.template, out); + } + if (ch === 92) { + // '\' + out += this.input.slice(chunkStart, this.state.pos); + out += this.readEscapedChar(true); + chunkStart = this.state.pos; + } else if ((0, _whitespace.isNewLine)(ch)) { + out += this.input.slice(chunkStart, this.state.pos); + ++this.state.pos; + switch (ch) { + case 13: + if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; + case 10: + out += "\n"; + break; + default: + out += String.fromCharCode(ch); + break; + } + ++this.state.curLine; + this.state.lineStart = this.state.pos; + chunkStart = this.state.pos; + } else { + ++this.state.pos; + } + } + }; + + // Used to read escaped characters + + Tokenizer.prototype.readEscapedChar = function readEscapedChar(inTemplate) { + var ch = this.input.charCodeAt(++this.state.pos); + ++this.state.pos; + switch (ch) { + case 110: + return "\n"; // 'n' -> '\n' + case 114: + return "\r"; // 'r' -> '\r' + case 120: + return String.fromCharCode(this.readHexChar(2)); // 'x' + case 117: + return codePointToString(this.readCodePoint()); // 'u' + case 116: + return "\t"; // 't' -> '\t' + case 98: + return "\b"; // 'b' -> '\b' + case 118: + return "\u000b"; // 'v' -> '\u000b' + case 102: + return "\f"; // 'f' -> '\f' + case 13: + if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; // '\r\n' + case 10: + // ' \n' + this.state.lineStart = this.state.pos; + ++this.state.curLine; + return ""; + default: + if (ch >= 48 && ch <= 55) { + var octalStr = this.input.substr(this.state.pos - 1, 3).match(/^[0-7]+/)[0]; + var octal = parseInt(octalStr, 8); + if (octal > 255) { + octalStr = octalStr.slice(0, -1); + octal = parseInt(octalStr, 8); + } + if (octal > 0) { + if (!this.state.containsOctal) { + this.state.containsOctal = true; + this.state.octalPosition = this.state.pos - 2; + } + if (this.state.strict || inTemplate) { + this.raise(this.state.pos - 2, "Octal literal in strict mode"); + } + } + this.state.pos += octalStr.length - 1; + return String.fromCharCode(octal); + } + return String.fromCharCode(ch); + } + }; + + // Used to read character escape sequences ('\x', '\u', '\U'). + + Tokenizer.prototype.readHexChar = function readHexChar(len) { + var codePos = this.state.pos; + var n = this.readInt(16, len); + if (n === null) this.raise(codePos, "Bad character escape sequence"); + return n; + }; + + // Read an identifier, and return it as a string. Sets `this.state.containsEsc` + // to whether the word contained a '\u' escape. + // + // Incrementally adds only escaped chars, adding other chunks as-is + // as a micro-optimization. + + Tokenizer.prototype.readWord1 = function readWord1() { + this.state.containsEsc = false; + var word = "", + first = true, + chunkStart = this.state.pos; + while (this.state.pos < this.input.length) { + var ch = this.fullCharCodeAtPos(); + if ((0, _identifier.isIdentifierChar)(ch)) { + this.state.pos += ch <= 0xffff ? 1 : 2; + } else if (ch === 92) { + // "\" + this.state.containsEsc = true; + + word += this.input.slice(chunkStart, this.state.pos); + var escStart = this.state.pos; + + if (this.input.charCodeAt(++this.state.pos) !== 117) { + // "u" + this.raise(this.state.pos, "Expecting Unicode escape sequence \\uXXXX"); + } + + ++this.state.pos; + var esc = this.readCodePoint(); + if (!(first ? _identifier.isIdentifierStart : _identifier.isIdentifierChar)(esc, true)) { + this.raise(escStart, "Invalid Unicode escape"); + } + + word += codePointToString(esc); + chunkStart = this.state.pos; + } else { + break; + } + first = false; + } + return word + this.input.slice(chunkStart, this.state.pos); + }; + + // Read an identifier or keyword token. Will check for reserved + // words when necessary. + + Tokenizer.prototype.readWord = function readWord() { + var word = this.readWord1(); + var type = _types.types.name; + if (!this.state.containsEsc && this.isKeyword(word)) { + type = _types.keywords[word]; + } + return this.finishToken(type, word); + }; + + Tokenizer.prototype.braceIsBlock = function braceIsBlock(prevType) { + if (prevType === _types.types.colon) { + var parent = this.curContext(); + if (parent === _context.types.braceStatement || parent === _context.types.braceExpression) { + return !parent.isExpr; + } + } + + if (prevType === _types.types._return) { + return _whitespace.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start)); + } + + if (prevType === _types.types._else || prevType === _types.types.semi || prevType === _types.types.eof || prevType === _types.types.parenR) { + return true; + } + + if (prevType === _types.types.braceL) { + return this.curContext() === _context.types.braceStatement; + } + + return !this.state.exprAllowed; + }; + + Tokenizer.prototype.updateContext = function updateContext(prevType) { + var update = void 0, + type = this.state.type; + if (type.keyword && prevType === _types.types.dot) { + this.state.exprAllowed = false; + } else if (update = type.updateContext) { + update.call(this, prevType); + } else { + this.state.exprAllowed = type.beforeExpr; + } + }; + + return Tokenizer; +}(); + +exports.default = Tokenizer;
\ No newline at end of file diff --git a/node_modules/babylon/lib/tokenizer/state.js b/node_modules/babylon/lib/tokenizer/state.js new file mode 100644 index 000000000..30a6a9e50 --- /dev/null +++ b/node_modules/babylon/lib/tokenizer/state.js @@ -0,0 +1,146 @@ +"use strict"; + +exports.__esModule = true; + +var _location = require("../util/location"); + +var _context = require("./context"); + +var _types = require("./types"); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var State = function () { + function State() { + _classCallCheck(this, State); + } + + State.prototype.init = function init(options, input) { + this.strict = options.strictMode === false ? false : options.sourceType === "module"; + + this.input = input; + + this.potentialArrowAt = -1; + + this.inMethod = this.inFunction = this.inGenerator = this.inAsync = false; + + this.labels = []; + + this.decorators = []; + + this.tokens = []; + + this.comments = []; + + this.trailingComments = []; + this.leadingComments = []; + this.commentStack = []; + + this.pos = this.lineStart = 0; + this.curLine = 1; + + this.type = _types.types.eof; + this.value = null; + this.start = this.end = this.pos; + this.startLoc = this.endLoc = this.curPosition(); + + this.lastTokEndLoc = this.lastTokStartLoc = null; + this.lastTokStart = this.lastTokEnd = this.pos; + + this.context = [_context.types.braceStatement]; + this.exprAllowed = true; + + this.containsEsc = this.containsOctal = false; + this.octalPosition = null; + + this.exportedIdentifiers = []; + + return this; + }; + + // TODO + + + // TODO + + + // Used to signify the start of a potential arrow function + + + // Flags to track whether we are in a function, a generator. + + + // Labels in scope. + + + // Leading decorators. + + + // Token store. + + + // Comment store. + + + // Comment attachment store + + + // The current position of the tokenizer in the input. + + + // Properties of the current token: + // Its type + + + // For tokens that include more information than their type, the value + + + // Its start and end offset + + + // And, if locations are used, the {line, column} object + // corresponding to those offsets + + + // Position information for the previous token + + + // The context stack is used to superficially track syntactic + // context to predict whether a regular expression is allowed in a + // given position. + + + // Used to signal to callers of `readWord1` whether the word + // contained any escape sequences. This is needed because words with + // escape sequences must not be interpreted as keywords. + + + // TODO + + + // Names of exports store. `default` is stored as a name for both + // `export default foo;` and `export { foo as default };`. + + + State.prototype.curPosition = function curPosition() { + return new _location.Position(this.curLine, this.pos - this.lineStart); + }; + + State.prototype.clone = function clone(skipArrays) { + var state = new State(); + for (var key in this) { + var val = this[key]; + + if ((!skipArrays || key === "context") && Array.isArray(val)) { + val = val.slice(); + } + + state[key] = val; + } + return state; + }; + + return State; +}(); + +exports.default = State;
\ No newline at end of file diff --git a/node_modules/babylon/lib/tokenizer/types.js b/node_modules/babylon/lib/tokenizer/types.js new file mode 100644 index 000000000..e9420feb3 --- /dev/null +++ b/node_modules/babylon/lib/tokenizer/types.js @@ -0,0 +1,159 @@ +"use strict"; + +exports.__esModule = true; + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +// ## Token types + +// The assignment of fine-grained, information-carrying type objects +// allows the tokenizer to store the information it has about a +// token in a way that is very cheap for the parser to look up. + +// All token type variables start with an underscore, to make them +// easy to recognize. + +// The `beforeExpr` property is used to disambiguate between regular +// expressions and divisions. It is set on all token types that can +// be followed by an expression (thus, a slash after them would be a +// regular expression). +// +// `isLoop` marks a keyword as starting a loop, which is important +// to know when parsing a label, in order to allow or disallow +// continue jumps to that label. + +var TokenType = exports.TokenType = function TokenType(label) { + var conf = arguments.length <= 1 || arguments[1] === undefined ? {} : arguments[1]; + + _classCallCheck(this, TokenType); + + this.label = label; + this.keyword = conf.keyword; + this.beforeExpr = !!conf.beforeExpr; + this.startsExpr = !!conf.startsExpr; + this.rightAssociative = !!conf.rightAssociative; + this.isLoop = !!conf.isLoop; + this.isAssign = !!conf.isAssign; + this.prefix = !!conf.prefix; + this.postfix = !!conf.postfix; + this.binop = conf.binop || null; + this.updateContext = null; +}; + +function binop(name, prec) { + return new TokenType(name, { beforeExpr: true, binop: prec }); +} +var beforeExpr = { beforeExpr: true }, + startsExpr = { startsExpr: true }; + +var types = exports.types = { + num: new TokenType("num", startsExpr), + regexp: new TokenType("regexp", startsExpr), + string: new TokenType("string", startsExpr), + name: new TokenType("name", startsExpr), + eof: new TokenType("eof"), + + // Punctuation token types. + bracketL: new TokenType("[", { beforeExpr: true, startsExpr: true }), + bracketR: new TokenType("]"), + braceL: new TokenType("{", { beforeExpr: true, startsExpr: true }), + braceBarL: new TokenType("{|", { beforeExpr: true, startsExpr: true }), + braceR: new TokenType("}"), + braceBarR: new TokenType("|}"), + parenL: new TokenType("(", { beforeExpr: true, startsExpr: true }), + parenR: new TokenType(")"), + comma: new TokenType(",", beforeExpr), + semi: new TokenType(";", beforeExpr), + colon: new TokenType(":", beforeExpr), + doubleColon: new TokenType("::", beforeExpr), + dot: new TokenType("."), + question: new TokenType("?", beforeExpr), + arrow: new TokenType("=>", beforeExpr), + template: new TokenType("template"), + ellipsis: new TokenType("...", beforeExpr), + backQuote: new TokenType("`", startsExpr), + dollarBraceL: new TokenType("${", { beforeExpr: true, startsExpr: true }), + at: new TokenType("@"), + + // Operators. These carry several kinds of properties to help the + // parser use them properly (the presence of these properties is + // what categorizes them as operators). + // + // `binop`, when present, specifies that this operator is a binary + // operator, and will refer to its precedence. + // + // `prefix` and `postfix` mark the operator as a prefix or postfix + // unary operator. + // + // `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as + // binary operators with a very low precedence, that should result + // in AssignmentExpression nodes. + + eq: new TokenType("=", { beforeExpr: true, isAssign: true }), + assign: new TokenType("_=", { beforeExpr: true, isAssign: true }), + incDec: new TokenType("++/--", { prefix: true, postfix: true, startsExpr: true }), + prefix: new TokenType("prefix", { beforeExpr: true, prefix: true, startsExpr: true }), + logicalOR: binop("||", 1), + logicalAND: binop("&&", 2), + bitwiseOR: binop("|", 3), + bitwiseXOR: binop("^", 4), + bitwiseAND: binop("&", 5), + equality: binop("==/!=", 6), + relational: binop("</>", 7), + bitShift: binop("<</>>", 8), + plusMin: new TokenType("+/-", { beforeExpr: true, binop: 9, prefix: true, startsExpr: true }), + modulo: binop("%", 10), + star: binop("*", 10), + slash: binop("/", 10), + exponent: new TokenType("**", { beforeExpr: true, binop: 11, rightAssociative: true }) +}; + +// Map keyword names to token types. + +var keywords = exports.keywords = {}; + +// Succinct definitions of keyword token types +function kw(name) { + var options = arguments.length <= 1 || arguments[1] === undefined ? {} : arguments[1]; + + options.keyword = name; + keywords[name] = types["_" + name] = new TokenType(name, options); +} + +kw("break"); +kw("case", beforeExpr); +kw("catch"); +kw("continue"); +kw("debugger"); +kw("default", beforeExpr); +kw("do", { isLoop: true, beforeExpr: true }); +kw("else", beforeExpr); +kw("finally"); +kw("for", { isLoop: true }); +kw("function", startsExpr); +kw("if"); +kw("return", beforeExpr); +kw("switch"); +kw("throw", beforeExpr); +kw("try"); +kw("var"); +kw("let"); +kw("const"); +kw("while", { isLoop: true }); +kw("with"); +kw("new", { beforeExpr: true, startsExpr: true }); +kw("this", startsExpr); +kw("super", startsExpr); +kw("class"); +kw("extends", beforeExpr); +kw("export"); +kw("import"); +kw("yield", { beforeExpr: true, startsExpr: true }); +kw("null", startsExpr); +kw("true", startsExpr); +kw("false", startsExpr); +kw("in", { beforeExpr: true, binop: 7 }); +kw("instanceof", { beforeExpr: true, binop: 7 }); +kw("typeof", { beforeExpr: true, prefix: true, startsExpr: true }); +kw("void", { beforeExpr: true, prefix: true, startsExpr: true }); +kw("delete", { beforeExpr: true, prefix: true, startsExpr: true });
\ No newline at end of file |