diff options
Diffstat (limited to 'node_modules/babylon/lib/tokenizer')
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/context.js | 99 | ||||
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/index.js | 889 | ||||
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/state.js | 146 | ||||
| -rw-r--r-- | node_modules/babylon/lib/tokenizer/types.js | 159 |
4 files changed, 0 insertions, 1293 deletions
diff --git a/node_modules/babylon/lib/tokenizer/context.js b/node_modules/babylon/lib/tokenizer/context.js deleted file mode 100644 index 856051a01..000000000 --- a/node_modules/babylon/lib/tokenizer/context.js +++ /dev/null @@ -1,99 +0,0 @@ -"use strict"; - -exports.__esModule = true; -exports.types = exports.TokContext = undefined; - -var _types = require("./types"); - -var _whitespace = require("../util/whitespace"); - -function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } // The algorithm used to determine whether a regexp can appear at a -// given point in the program is loosely based on sweet.js' approach. -// See https://github.com/mozilla/sweet.js/wiki/design - -var TokContext = exports.TokContext = function TokContext(token, isExpr, preserveSpace, override) { - _classCallCheck(this, TokContext); - - this.token = token; - this.isExpr = !!isExpr; - this.preserveSpace = !!preserveSpace; - this.override = override; -}; - -var types = exports.types = { - braceStatement: new TokContext("{", false), - braceExpression: new TokContext("{", true), - templateQuasi: new TokContext("${", true), - parenStatement: new TokContext("(", false), - parenExpression: new TokContext("(", true), - template: new TokContext("`", true, true, function (p) { - return p.readTmplToken(); - }), - functionExpression: new TokContext("function", true) -}; - -// Token-specific context update code - -_types.types.parenR.updateContext = _types.types.braceR.updateContext = function () { - if (this.state.context.length === 1) { - this.state.exprAllowed = true; - return; - } - - var out = this.state.context.pop(); - if (out === types.braceStatement && this.curContext() === types.functionExpression) { - this.state.context.pop(); - this.state.exprAllowed = false; - } else if (out === types.templateQuasi) { - this.state.exprAllowed = true; - } else { - this.state.exprAllowed = !out.isExpr; - } -}; - -_types.types.name.updateContext = function (prevType) { - this.state.exprAllowed = false; - - if (prevType === _types.types._let || prevType === _types.types._const || prevType === _types.types._var) { - if (_whitespace.lineBreak.test(this.input.slice(this.state.end))) { - this.state.exprAllowed = true; - } - } -}; - -_types.types.braceL.updateContext = function (prevType) { - this.state.context.push(this.braceIsBlock(prevType) ? types.braceStatement : types.braceExpression); - this.state.exprAllowed = true; -}; - -_types.types.dollarBraceL.updateContext = function () { - this.state.context.push(types.templateQuasi); - this.state.exprAllowed = true; -}; - -_types.types.parenL.updateContext = function (prevType) { - var statementParens = prevType === _types.types._if || prevType === _types.types._for || prevType === _types.types._with || prevType === _types.types._while; - this.state.context.push(statementParens ? types.parenStatement : types.parenExpression); - this.state.exprAllowed = true; -}; - -_types.types.incDec.updateContext = function () { - // tokExprAllowed stays unchanged -}; - -_types.types._function.updateContext = function () { - if (this.curContext() !== types.braceStatement) { - this.state.context.push(types.functionExpression); - } - - this.state.exprAllowed = false; -}; - -_types.types.backQuote.updateContext = function () { - if (this.curContext() === types.template) { - this.state.context.pop(); - } else { - this.state.context.push(types.template); - } - this.state.exprAllowed = false; -};
\ No newline at end of file diff --git a/node_modules/babylon/lib/tokenizer/index.js b/node_modules/babylon/lib/tokenizer/index.js deleted file mode 100644 index 25d2b8d13..000000000 --- a/node_modules/babylon/lib/tokenizer/index.js +++ /dev/null @@ -1,889 +0,0 @@ -"use strict"; - -exports.__esModule = true; -exports.Token = undefined; - -var _identifier = require("../util/identifier"); - -var _types = require("./types"); - -var _context = require("./context"); - -var _location = require("../util/location"); - -var _whitespace = require("../util/whitespace"); - -var _state = require("./state"); - -var _state2 = _interopRequireDefault(_state); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } /* eslint max-len: 0 */ -/* eslint indent: 0 */ - -// Object type used to represent tokens. Note that normally, tokens -// simply exist as properties on the parser object. This is only -// used for the onToken callback and the external tokenizer. - -var Token = exports.Token = function Token(state) { - _classCallCheck(this, Token); - - this.type = state.type; - this.value = state.value; - this.start = state.start; - this.end = state.end; - this.loc = new _location.SourceLocation(state.startLoc, state.endLoc); -}; - -// ## Tokenizer - -function codePointToString(code) { - // UTF-16 Decoding - if (code <= 0xFFFF) { - return String.fromCharCode(code); - } else { - return String.fromCharCode((code - 0x10000 >> 10) + 0xD800, (code - 0x10000 & 1023) + 0xDC00); - } -} - -var Tokenizer = function () { - function Tokenizer(options, input) { - _classCallCheck(this, Tokenizer); - - this.state = new _state2.default(); - this.state.init(options, input); - } - - // Move to the next token - - Tokenizer.prototype.next = function next() { - if (!this.isLookahead) { - this.state.tokens.push(new Token(this.state)); - } - - this.state.lastTokEnd = this.state.end; - this.state.lastTokStart = this.state.start; - this.state.lastTokEndLoc = this.state.endLoc; - this.state.lastTokStartLoc = this.state.startLoc; - this.nextToken(); - }; - - // TODO - - Tokenizer.prototype.eat = function eat(type) { - if (this.match(type)) { - this.next(); - return true; - } else { - return false; - } - }; - - // TODO - - Tokenizer.prototype.match = function match(type) { - return this.state.type === type; - }; - - // TODO - - Tokenizer.prototype.isKeyword = function isKeyword(word) { - return (0, _identifier.isKeyword)(word); - }; - - // TODO - - Tokenizer.prototype.lookahead = function lookahead() { - var old = this.state; - this.state = old.clone(true); - - this.isLookahead = true; - this.next(); - this.isLookahead = false; - - var curr = this.state.clone(true); - this.state = old; - return curr; - }; - - // Toggle strict mode. Re-reads the next number or string to please - // pedantic tests (`"use strict"; 010;` should fail). - - Tokenizer.prototype.setStrict = function setStrict(strict) { - this.state.strict = strict; - if (!this.match(_types.types.num) && !this.match(_types.types.string)) return; - this.state.pos = this.state.start; - while (this.state.pos < this.state.lineStart) { - this.state.lineStart = this.input.lastIndexOf("\n", this.state.lineStart - 2) + 1; - --this.state.curLine; - } - this.nextToken(); - }; - - Tokenizer.prototype.curContext = function curContext() { - return this.state.context[this.state.context.length - 1]; - }; - - // Read a single token, updating the parser object's token-related - // properties. - - Tokenizer.prototype.nextToken = function nextToken() { - var curContext = this.curContext(); - if (!curContext || !curContext.preserveSpace) this.skipSpace(); - - this.state.containsOctal = false; - this.state.octalPosition = null; - this.state.start = this.state.pos; - this.state.startLoc = this.state.curPosition(); - if (this.state.pos >= this.input.length) return this.finishToken(_types.types.eof); - - if (curContext.override) { - return curContext.override(this); - } else { - return this.readToken(this.fullCharCodeAtPos()); - } - }; - - Tokenizer.prototype.readToken = function readToken(code) { - // Identifier or keyword. '\uXXXX' sequences are allowed in - // identifiers, so '\' also dispatches to that. - if ((0, _identifier.isIdentifierStart)(code) || code === 92 /* '\' */) { - return this.readWord(); - } else { - return this.getTokenFromCode(code); - } - }; - - Tokenizer.prototype.fullCharCodeAtPos = function fullCharCodeAtPos() { - var code = this.input.charCodeAt(this.state.pos); - if (code <= 0xd7ff || code >= 0xe000) return code; - - var next = this.input.charCodeAt(this.state.pos + 1); - return (code << 10) + next - 0x35fdc00; - }; - - Tokenizer.prototype.pushComment = function pushComment(block, text, start, end, startLoc, endLoc) { - var comment = { - type: block ? "CommentBlock" : "CommentLine", - value: text, - start: start, - end: end, - loc: new _location.SourceLocation(startLoc, endLoc) - }; - - if (!this.isLookahead) { - this.state.tokens.push(comment); - this.state.comments.push(comment); - this.addComment(comment); - } - }; - - Tokenizer.prototype.skipBlockComment = function skipBlockComment() { - var startLoc = this.state.curPosition(); - var start = this.state.pos, - end = this.input.indexOf("*/", this.state.pos += 2); - if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment"); - - this.state.pos = end + 2; - _whitespace.lineBreakG.lastIndex = start; - var match = void 0; - while ((match = _whitespace.lineBreakG.exec(this.input)) && match.index < this.state.pos) { - ++this.state.curLine; - this.state.lineStart = match.index + match[0].length; - } - - this.pushComment(true, this.input.slice(start + 2, end), start, this.state.pos, startLoc, this.state.curPosition()); - }; - - Tokenizer.prototype.skipLineComment = function skipLineComment(startSkip) { - var start = this.state.pos; - var startLoc = this.state.curPosition(); - var ch = this.input.charCodeAt(this.state.pos += startSkip); - while (this.state.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) { - ++this.state.pos; - ch = this.input.charCodeAt(this.state.pos); - } - - this.pushComment(false, this.input.slice(start + startSkip, this.state.pos), start, this.state.pos, startLoc, this.state.curPosition()); - }; - - // Called at the start of the parse and after every token. Skips - // whitespace and comments, and. - - Tokenizer.prototype.skipSpace = function skipSpace() { - loop: while (this.state.pos < this.input.length) { - var ch = this.input.charCodeAt(this.state.pos); - switch (ch) { - case 32:case 160: - // ' ' - ++this.state.pos; - break; - - case 13: - if (this.input.charCodeAt(this.state.pos + 1) === 10) { - ++this.state.pos; - } - - case 10:case 8232:case 8233: - ++this.state.pos; - ++this.state.curLine; - this.state.lineStart = this.state.pos; - break; - - case 47: - // '/' - switch (this.input.charCodeAt(this.state.pos + 1)) { - case 42: - // '*' - this.skipBlockComment(); - break; - - case 47: - this.skipLineComment(2); - break; - - default: - break loop; - } - break; - - default: - if (ch > 8 && ch < 14 || ch >= 5760 && _whitespace.nonASCIIwhitespace.test(String.fromCharCode(ch))) { - ++this.state.pos; - } else { - break loop; - } - } - } - }; - - // Called at the end of every token. Sets `end`, `val`, and - // maintains `context` and `exprAllowed`, and skips the space after - // the token, so that the next one's `start` will point at the - // right position. - - Tokenizer.prototype.finishToken = function finishToken(type, val) { - this.state.end = this.state.pos; - this.state.endLoc = this.state.curPosition(); - var prevType = this.state.type; - this.state.type = type; - this.state.value = val; - - this.updateContext(prevType); - }; - - // ### Token reading - - // This is the function that is called to fetch the next token. It - // is somewhat obscure, because it works in character codes rather - // than characters, and because operator parsing has been inlined - // into it. - // - // All in the name of speed. - // - - - Tokenizer.prototype.readToken_dot = function readToken_dot() { - var next = this.input.charCodeAt(this.state.pos + 1); - if (next >= 48 && next <= 57) { - return this.readNumber(true); - } - - var next2 = this.input.charCodeAt(this.state.pos + 2); - if (next === 46 && next2 === 46) { - // 46 = dot '.' - this.state.pos += 3; - return this.finishToken(_types.types.ellipsis); - } else { - ++this.state.pos; - return this.finishToken(_types.types.dot); - } - }; - - Tokenizer.prototype.readToken_slash = function readToken_slash() { - // '/' - if (this.state.exprAllowed) { - ++this.state.pos; - return this.readRegexp(); - } - - var next = this.input.charCodeAt(this.state.pos + 1); - if (next === 61) { - return this.finishOp(_types.types.assign, 2); - } else { - return this.finishOp(_types.types.slash, 1); - } - }; - - Tokenizer.prototype.readToken_mult_modulo = function readToken_mult_modulo(code) { - // '%*' - var type = code === 42 ? _types.types.star : _types.types.modulo; - var width = 1; - var next = this.input.charCodeAt(this.state.pos + 1); - - if (next === 42) { - // '*' - width++; - next = this.input.charCodeAt(this.state.pos + 2); - type = _types.types.exponent; - } - - if (next === 61) { - width++; - type = _types.types.assign; - } - - return this.finishOp(type, width); - }; - - Tokenizer.prototype.readToken_pipe_amp = function readToken_pipe_amp(code) { - // '|&' - var next = this.input.charCodeAt(this.state.pos + 1); - if (next === code) return this.finishOp(code === 124 ? _types.types.logicalOR : _types.types.logicalAND, 2); - if (next === 61) return this.finishOp(_types.types.assign, 2); - if (code === 124 && next === 125 && this.hasPlugin("flow")) return this.finishOp(_types.types.braceBarR, 2); - return this.finishOp(code === 124 ? _types.types.bitwiseOR : _types.types.bitwiseAND, 1); - }; - - Tokenizer.prototype.readToken_caret = function readToken_caret() { - // '^' - var next = this.input.charCodeAt(this.state.pos + 1); - if (next === 61) { - return this.finishOp(_types.types.assign, 2); - } else { - return this.finishOp(_types.types.bitwiseXOR, 1); - } - }; - - Tokenizer.prototype.readToken_plus_min = function readToken_plus_min(code) { - // '+-' - var next = this.input.charCodeAt(this.state.pos + 1); - - if (next === code) { - if (next === 45 && this.input.charCodeAt(this.state.pos + 2) === 62 && _whitespace.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))) { - // A `-->` line comment - this.skipLineComment(3); - this.skipSpace(); - return this.nextToken(); - } - return this.finishOp(_types.types.incDec, 2); - } - - if (next === 61) { - return this.finishOp(_types.types.assign, 2); - } else { - return this.finishOp(_types.types.plusMin, 1); - } - }; - - Tokenizer.prototype.readToken_lt_gt = function readToken_lt_gt(code) { - // '<>' - var next = this.input.charCodeAt(this.state.pos + 1); - var size = 1; - - if (next === code) { - size = code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2; - if (this.input.charCodeAt(this.state.pos + size) === 61) return this.finishOp(_types.types.assign, size + 1); - return this.finishOp(_types.types.bitShift, size); - } - - if (next === 33 && code === 60 && this.input.charCodeAt(this.state.pos + 2) === 45 && this.input.charCodeAt(this.state.pos + 3) === 45) { - if (this.inModule) this.unexpected(); - // `<!--`, an XML-style comment that should be interpreted as a line comment - this.skipLineComment(4); - this.skipSpace(); - return this.nextToken(); - } - - if (next === 61) { - // <= | >= - size = 2; - } - - return this.finishOp(_types.types.relational, size); - }; - - Tokenizer.prototype.readToken_eq_excl = function readToken_eq_excl(code) { - // '=!' - var next = this.input.charCodeAt(this.state.pos + 1); - if (next === 61) return this.finishOp(_types.types.equality, this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2); - if (code === 61 && next === 62) { - // '=>' - this.state.pos += 2; - return this.finishToken(_types.types.arrow); - } - return this.finishOp(code === 61 ? _types.types.eq : _types.types.prefix, 1); - }; - - Tokenizer.prototype.getTokenFromCode = function getTokenFromCode(code) { - switch (code) { - // The interpretation of a dot depends on whether it is followed - // by a digit or another two dots. - case 46: - // '.' - return this.readToken_dot(); - - // Punctuation tokens. - case 40: - ++this.state.pos;return this.finishToken(_types.types.parenL); - case 41: - ++this.state.pos;return this.finishToken(_types.types.parenR); - case 59: - ++this.state.pos;return this.finishToken(_types.types.semi); - case 44: - ++this.state.pos;return this.finishToken(_types.types.comma); - case 91: - ++this.state.pos;return this.finishToken(_types.types.bracketL); - case 93: - ++this.state.pos;return this.finishToken(_types.types.bracketR); - - case 123: - if (this.hasPlugin("flow") && this.input.charCodeAt(this.state.pos + 1) === 124) { - return this.finishOp(_types.types.braceBarL, 2); - } else { - ++this.state.pos; - return this.finishToken(_types.types.braceL); - } - - case 125: - ++this.state.pos;return this.finishToken(_types.types.braceR); - - case 58: - if (this.hasPlugin("functionBind") && this.input.charCodeAt(this.state.pos + 1) === 58) { - return this.finishOp(_types.types.doubleColon, 2); - } else { - ++this.state.pos; - return this.finishToken(_types.types.colon); - } - - case 63: - ++this.state.pos;return this.finishToken(_types.types.question); - case 64: - ++this.state.pos;return this.finishToken(_types.types.at); - - case 96: - // '`' - ++this.state.pos; - return this.finishToken(_types.types.backQuote); - - case 48: - // '0' - var next = this.input.charCodeAt(this.state.pos + 1); - if (next === 120 || next === 88) return this.readRadixNumber(16); // '0x', '0X' - hex number - if (next === 111 || next === 79) return this.readRadixNumber(8); // '0o', '0O' - octal number - if (next === 98 || next === 66) return this.readRadixNumber(2); // '0b', '0B' - binary number - // Anything else beginning with a digit is an integer, octal - // number, or float. - case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57: - // 1-9 - return this.readNumber(false); - - // Quotes produce strings. - case 34:case 39: - // '"', "'" - return this.readString(code); - - // Operators are parsed inline in tiny state machines. '=' (61) is - // often referred to. `finishOp` simply skips the amount of - // characters it is given as second argument, and returns a token - // of the type given by its first argument. - - case 47: - // '/' - return this.readToken_slash(); - - case 37:case 42: - // '%*' - return this.readToken_mult_modulo(code); - - case 124:case 38: - // '|&' - return this.readToken_pipe_amp(code); - - case 94: - // '^' - return this.readToken_caret(); - - case 43:case 45: - // '+-' - return this.readToken_plus_min(code); - - case 60:case 62: - // '<>' - return this.readToken_lt_gt(code); - - case 61:case 33: - // '=!' - return this.readToken_eq_excl(code); - - case 126: - // '~' - return this.finishOp(_types.types.prefix, 1); - } - - this.raise(this.state.pos, "Unexpected character '" + codePointToString(code) + "'"); - }; - - Tokenizer.prototype.finishOp = function finishOp(type, size) { - var str = this.input.slice(this.state.pos, this.state.pos + size); - this.state.pos += size; - return this.finishToken(type, str); - }; - - Tokenizer.prototype.readRegexp = function readRegexp() { - var escaped = void 0, - inClass = void 0, - start = this.state.pos; - for (;;) { - if (this.state.pos >= this.input.length) this.raise(start, "Unterminated regular expression"); - var ch = this.input.charAt(this.state.pos); - if (_whitespace.lineBreak.test(ch)) { - this.raise(start, "Unterminated regular expression"); - } - if (escaped) { - escaped = false; - } else { - if (ch === "[") { - inClass = true; - } else if (ch === "]" && inClass) { - inClass = false; - } else if (ch === "/" && !inClass) { - break; - } - escaped = ch === "\\"; - } - ++this.state.pos; - } - var content = this.input.slice(start, this.state.pos); - ++this.state.pos; - // Need to use `readWord1` because '\uXXXX' sequences are allowed - // here (don't ask). - var mods = this.readWord1(); - if (mods) { - var validFlags = /^[gmsiyu]*$/; - if (!validFlags.test(mods)) this.raise(start, "Invalid regular expression flag"); - } - return this.finishToken(_types.types.regexp, { - pattern: content, - flags: mods - }); - }; - - // Read an integer in the given radix. Return null if zero digits - // were read, the integer value otherwise. When `len` is given, this - // will return `null` unless the integer has exactly `len` digits. - - Tokenizer.prototype.readInt = function readInt(radix, len) { - var start = this.state.pos, - total = 0; - for (var i = 0, e = len == null ? Infinity : len; i < e; ++i) { - var code = this.input.charCodeAt(this.state.pos), - val = void 0; - if (code >= 97) { - val = code - 97 + 10; // a - } else if (code >= 65) { - val = code - 65 + 10; // A - } else if (code >= 48 && code <= 57) { - val = code - 48; // 0-9 - } else { - val = Infinity; - } - if (val >= radix) break; - ++this.state.pos; - total = total * radix + val; - } - if (this.state.pos === start || len != null && this.state.pos - start !== len) return null; - - return total; - }; - - Tokenizer.prototype.readRadixNumber = function readRadixNumber(radix) { - this.state.pos += 2; // 0x - var val = this.readInt(radix); - if (val == null) this.raise(this.state.start + 2, "Expected number in radix " + radix); - if ((0, _identifier.isIdentifierStart)(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number"); - return this.finishToken(_types.types.num, val); - }; - - // Read an integer, octal integer, or floating-point number. - - Tokenizer.prototype.readNumber = function readNumber(startsWithDot) { - var start = this.state.pos, - isFloat = false, - octal = this.input.charCodeAt(this.state.pos) === 48; - if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number"); - var next = this.input.charCodeAt(this.state.pos); - if (next === 46) { - // '.' - ++this.state.pos; - this.readInt(10); - isFloat = true; - next = this.input.charCodeAt(this.state.pos); - } - if (next === 69 || next === 101) { - // 'eE' - next = this.input.charCodeAt(++this.state.pos); - if (next === 43 || next === 45) ++this.state.pos; // '+-' - if (this.readInt(10) === null) this.raise(start, "Invalid number"); - isFloat = true; - } - if ((0, _identifier.isIdentifierStart)(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number"); - - var str = this.input.slice(start, this.state.pos), - val = void 0; - if (isFloat) { - val = parseFloat(str); - } else if (!octal || str.length === 1) { - val = parseInt(str, 10); - } else if (/[89]/.test(str) || this.state.strict) { - this.raise(start, "Invalid number"); - } else { - val = parseInt(str, 8); - } - return this.finishToken(_types.types.num, val); - }; - - // Read a string value, interpreting backslash-escapes. - - Tokenizer.prototype.readCodePoint = function readCodePoint() { - var ch = this.input.charCodeAt(this.state.pos), - code = void 0; - - if (ch === 123) { - var codePos = ++this.state.pos; - code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos); - ++this.state.pos; - if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds"); - } else { - code = this.readHexChar(4); - } - return code; - }; - - Tokenizer.prototype.readString = function readString(quote) { - var out = "", - chunkStart = ++this.state.pos; - for (;;) { - if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated string constant"); - var ch = this.input.charCodeAt(this.state.pos); - if (ch === quote) break; - if (ch === 92) { - // '\' - out += this.input.slice(chunkStart, this.state.pos); - out += this.readEscapedChar(false); - chunkStart = this.state.pos; - } else { - if ((0, _whitespace.isNewLine)(ch)) this.raise(this.state.start, "Unterminated string constant"); - ++this.state.pos; - } - } - out += this.input.slice(chunkStart, this.state.pos++); - return this.finishToken(_types.types.string, out); - }; - - // Reads template string tokens. - - Tokenizer.prototype.readTmplToken = function readTmplToken() { - var out = "", - chunkStart = this.state.pos; - for (;;) { - if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated template"); - var ch = this.input.charCodeAt(this.state.pos); - if (ch === 96 || ch === 36 && this.input.charCodeAt(this.state.pos + 1) === 123) { - // '`', '${' - if (this.state.pos === this.state.start && this.match(_types.types.template)) { - if (ch === 36) { - this.state.pos += 2; - return this.finishToken(_types.types.dollarBraceL); - } else { - ++this.state.pos; - return this.finishToken(_types.types.backQuote); - } - } - out += this.input.slice(chunkStart, this.state.pos); - return this.finishToken(_types.types.template, out); - } - if (ch === 92) { - // '\' - out += this.input.slice(chunkStart, this.state.pos); - out += this.readEscapedChar(true); - chunkStart = this.state.pos; - } else if ((0, _whitespace.isNewLine)(ch)) { - out += this.input.slice(chunkStart, this.state.pos); - ++this.state.pos; - switch (ch) { - case 13: - if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; - case 10: - out += "\n"; - break; - default: - out += String.fromCharCode(ch); - break; - } - ++this.state.curLine; - this.state.lineStart = this.state.pos; - chunkStart = this.state.pos; - } else { - ++this.state.pos; - } - } - }; - - // Used to read escaped characters - - Tokenizer.prototype.readEscapedChar = function readEscapedChar(inTemplate) { - var ch = this.input.charCodeAt(++this.state.pos); - ++this.state.pos; - switch (ch) { - case 110: - return "\n"; // 'n' -> '\n' - case 114: - return "\r"; // 'r' -> '\r' - case 120: - return String.fromCharCode(this.readHexChar(2)); // 'x' - case 117: - return codePointToString(this.readCodePoint()); // 'u' - case 116: - return "\t"; // 't' -> '\t' - case 98: - return "\b"; // 'b' -> '\b' - case 118: - return "\u000b"; // 'v' -> '\u000b' - case 102: - return "\f"; // 'f' -> '\f' - case 13: - if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; // '\r\n' - case 10: - // ' \n' - this.state.lineStart = this.state.pos; - ++this.state.curLine; - return ""; - default: - if (ch >= 48 && ch <= 55) { - var octalStr = this.input.substr(this.state.pos - 1, 3).match(/^[0-7]+/)[0]; - var octal = parseInt(octalStr, 8); - if (octal > 255) { - octalStr = octalStr.slice(0, -1); - octal = parseInt(octalStr, 8); - } - if (octal > 0) { - if (!this.state.containsOctal) { - this.state.containsOctal = true; - this.state.octalPosition = this.state.pos - 2; - } - if (this.state.strict || inTemplate) { - this.raise(this.state.pos - 2, "Octal literal in strict mode"); - } - } - this.state.pos += octalStr.length - 1; - return String.fromCharCode(octal); - } - return String.fromCharCode(ch); - } - }; - - // Used to read character escape sequences ('\x', '\u', '\U'). - - Tokenizer.prototype.readHexChar = function readHexChar(len) { - var codePos = this.state.pos; - var n = this.readInt(16, len); - if (n === null) this.raise(codePos, "Bad character escape sequence"); - return n; - }; - - // Read an identifier, and return it as a string. Sets `this.state.containsEsc` - // to whether the word contained a '\u' escape. - // - // Incrementally adds only escaped chars, adding other chunks as-is - // as a micro-optimization. - - Tokenizer.prototype.readWord1 = function readWord1() { - this.state.containsEsc = false; - var word = "", - first = true, - chunkStart = this.state.pos; - while (this.state.pos < this.input.length) { - var ch = this.fullCharCodeAtPos(); - if ((0, _identifier.isIdentifierChar)(ch)) { - this.state.pos += ch <= 0xffff ? 1 : 2; - } else if (ch === 92) { - // "\" - this.state.containsEsc = true; - - word += this.input.slice(chunkStart, this.state.pos); - var escStart = this.state.pos; - - if (this.input.charCodeAt(++this.state.pos) !== 117) { - // "u" - this.raise(this.state.pos, "Expecting Unicode escape sequence \\uXXXX"); - } - - ++this.state.pos; - var esc = this.readCodePoint(); - if (!(first ? _identifier.isIdentifierStart : _identifier.isIdentifierChar)(esc, true)) { - this.raise(escStart, "Invalid Unicode escape"); - } - - word += codePointToString(esc); - chunkStart = this.state.pos; - } else { - break; - } - first = false; - } - return word + this.input.slice(chunkStart, this.state.pos); - }; - - // Read an identifier or keyword token. Will check for reserved - // words when necessary. - - Tokenizer.prototype.readWord = function readWord() { - var word = this.readWord1(); - var type = _types.types.name; - if (!this.state.containsEsc && this.isKeyword(word)) { - type = _types.keywords[word]; - } - return this.finishToken(type, word); - }; - - Tokenizer.prototype.braceIsBlock = function braceIsBlock(prevType) { - if (prevType === _types.types.colon) { - var parent = this.curContext(); - if (parent === _context.types.braceStatement || parent === _context.types.braceExpression) { - return !parent.isExpr; - } - } - - if (prevType === _types.types._return) { - return _whitespace.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start)); - } - - if (prevType === _types.types._else || prevType === _types.types.semi || prevType === _types.types.eof || prevType === _types.types.parenR) { - return true; - } - - if (prevType === _types.types.braceL) { - return this.curContext() === _context.types.braceStatement; - } - - return !this.state.exprAllowed; - }; - - Tokenizer.prototype.updateContext = function updateContext(prevType) { - var update = void 0, - type = this.state.type; - if (type.keyword && prevType === _types.types.dot) { - this.state.exprAllowed = false; - } else if (update = type.updateContext) { - update.call(this, prevType); - } else { - this.state.exprAllowed = type.beforeExpr; - } - }; - - return Tokenizer; -}(); - -exports.default = Tokenizer;
\ No newline at end of file diff --git a/node_modules/babylon/lib/tokenizer/state.js b/node_modules/babylon/lib/tokenizer/state.js deleted file mode 100644 index 30a6a9e50..000000000 --- a/node_modules/babylon/lib/tokenizer/state.js +++ /dev/null @@ -1,146 +0,0 @@ -"use strict"; - -exports.__esModule = true; - -var _location = require("../util/location"); - -var _context = require("./context"); - -var _types = require("./types"); - -function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } - -var State = function () { - function State() { - _classCallCheck(this, State); - } - - State.prototype.init = function init(options, input) { - this.strict = options.strictMode === false ? false : options.sourceType === "module"; - - this.input = input; - - this.potentialArrowAt = -1; - - this.inMethod = this.inFunction = this.inGenerator = this.inAsync = false; - - this.labels = []; - - this.decorators = []; - - this.tokens = []; - - this.comments = []; - - this.trailingComments = []; - this.leadingComments = []; - this.commentStack = []; - - this.pos = this.lineStart = 0; - this.curLine = 1; - - this.type = _types.types.eof; - this.value = null; - this.start = this.end = this.pos; - this.startLoc = this.endLoc = this.curPosition(); - - this.lastTokEndLoc = this.lastTokStartLoc = null; - this.lastTokStart = this.lastTokEnd = this.pos; - - this.context = [_context.types.braceStatement]; - this.exprAllowed = true; - - this.containsEsc = this.containsOctal = false; - this.octalPosition = null; - - this.exportedIdentifiers = []; - - return this; - }; - - // TODO - - - // TODO - - - // Used to signify the start of a potential arrow function - - - // Flags to track whether we are in a function, a generator. - - - // Labels in scope. - - - // Leading decorators. - - - // Token store. - - - // Comment store. - - - // Comment attachment store - - - // The current position of the tokenizer in the input. - - - // Properties of the current token: - // Its type - - - // For tokens that include more information than their type, the value - - - // Its start and end offset - - - // And, if locations are used, the {line, column} object - // corresponding to those offsets - - - // Position information for the previous token - - - // The context stack is used to superficially track syntactic - // context to predict whether a regular expression is allowed in a - // given position. - - - // Used to signal to callers of `readWord1` whether the word - // contained any escape sequences. This is needed because words with - // escape sequences must not be interpreted as keywords. - - - // TODO - - - // Names of exports store. `default` is stored as a name for both - // `export default foo;` and `export { foo as default };`. - - - State.prototype.curPosition = function curPosition() { - return new _location.Position(this.curLine, this.pos - this.lineStart); - }; - - State.prototype.clone = function clone(skipArrays) { - var state = new State(); - for (var key in this) { - var val = this[key]; - - if ((!skipArrays || key === "context") && Array.isArray(val)) { - val = val.slice(); - } - - state[key] = val; - } - return state; - }; - - return State; -}(); - -exports.default = State;
\ No newline at end of file diff --git a/node_modules/babylon/lib/tokenizer/types.js b/node_modules/babylon/lib/tokenizer/types.js deleted file mode 100644 index e9420feb3..000000000 --- a/node_modules/babylon/lib/tokenizer/types.js +++ /dev/null @@ -1,159 +0,0 @@ -"use strict"; - -exports.__esModule = true; - -function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } - -// ## Token types - -// The assignment of fine-grained, information-carrying type objects -// allows the tokenizer to store the information it has about a -// token in a way that is very cheap for the parser to look up. - -// All token type variables start with an underscore, to make them -// easy to recognize. - -// The `beforeExpr` property is used to disambiguate between regular -// expressions and divisions. It is set on all token types that can -// be followed by an expression (thus, a slash after them would be a -// regular expression). -// -// `isLoop` marks a keyword as starting a loop, which is important -// to know when parsing a label, in order to allow or disallow -// continue jumps to that label. - -var TokenType = exports.TokenType = function TokenType(label) { - var conf = arguments.length <= 1 || arguments[1] === undefined ? {} : arguments[1]; - - _classCallCheck(this, TokenType); - - this.label = label; - this.keyword = conf.keyword; - this.beforeExpr = !!conf.beforeExpr; - this.startsExpr = !!conf.startsExpr; - this.rightAssociative = !!conf.rightAssociative; - this.isLoop = !!conf.isLoop; - this.isAssign = !!conf.isAssign; - this.prefix = !!conf.prefix; - this.postfix = !!conf.postfix; - this.binop = conf.binop || null; - this.updateContext = null; -}; - -function binop(name, prec) { - return new TokenType(name, { beforeExpr: true, binop: prec }); -} -var beforeExpr = { beforeExpr: true }, - startsExpr = { startsExpr: true }; - -var types = exports.types = { - num: new TokenType("num", startsExpr), - regexp: new TokenType("regexp", startsExpr), - string: new TokenType("string", startsExpr), - name: new TokenType("name", startsExpr), - eof: new TokenType("eof"), - - // Punctuation token types. - bracketL: new TokenType("[", { beforeExpr: true, startsExpr: true }), - bracketR: new TokenType("]"), - braceL: new TokenType("{", { beforeExpr: true, startsExpr: true }), - braceBarL: new TokenType("{|", { beforeExpr: true, startsExpr: true }), - braceR: new TokenType("}"), - braceBarR: new TokenType("|}"), - parenL: new TokenType("(", { beforeExpr: true, startsExpr: true }), - parenR: new TokenType(")"), - comma: new TokenType(",", beforeExpr), - semi: new TokenType(";", beforeExpr), - colon: new TokenType(":", beforeExpr), - doubleColon: new TokenType("::", beforeExpr), - dot: new TokenType("."), - question: new TokenType("?", beforeExpr), - arrow: new TokenType("=>", beforeExpr), - template: new TokenType("template"), - ellipsis: new TokenType("...", beforeExpr), - backQuote: new TokenType("`", startsExpr), - dollarBraceL: new TokenType("${", { beforeExpr: true, startsExpr: true }), - at: new TokenType("@"), - - // Operators. These carry several kinds of properties to help the - // parser use them properly (the presence of these properties is - // what categorizes them as operators). - // - // `binop`, when present, specifies that this operator is a binary - // operator, and will refer to its precedence. - // - // `prefix` and `postfix` mark the operator as a prefix or postfix - // unary operator. - // - // `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as - // binary operators with a very low precedence, that should result - // in AssignmentExpression nodes. - - eq: new TokenType("=", { beforeExpr: true, isAssign: true }), - assign: new TokenType("_=", { beforeExpr: true, isAssign: true }), - incDec: new TokenType("++/--", { prefix: true, postfix: true, startsExpr: true }), - prefix: new TokenType("prefix", { beforeExpr: true, prefix: true, startsExpr: true }), - logicalOR: binop("||", 1), - logicalAND: binop("&&", 2), - bitwiseOR: binop("|", 3), - bitwiseXOR: binop("^", 4), - bitwiseAND: binop("&", 5), - equality: binop("==/!=", 6), - relational: binop("</>", 7), - bitShift: binop("<</>>", 8), - plusMin: new TokenType("+/-", { beforeExpr: true, binop: 9, prefix: true, startsExpr: true }), - modulo: binop("%", 10), - star: binop("*", 10), - slash: binop("/", 10), - exponent: new TokenType("**", { beforeExpr: true, binop: 11, rightAssociative: true }) -}; - -// Map keyword names to token types. - -var keywords = exports.keywords = {}; - -// Succinct definitions of keyword token types -function kw(name) { - var options = arguments.length <= 1 || arguments[1] === undefined ? {} : arguments[1]; - - options.keyword = name; - keywords[name] = types["_" + name] = new TokenType(name, options); -} - -kw("break"); -kw("case", beforeExpr); -kw("catch"); -kw("continue"); -kw("debugger"); -kw("default", beforeExpr); -kw("do", { isLoop: true, beforeExpr: true }); -kw("else", beforeExpr); -kw("finally"); -kw("for", { isLoop: true }); -kw("function", startsExpr); -kw("if"); -kw("return", beforeExpr); -kw("switch"); -kw("throw", beforeExpr); -kw("try"); -kw("var"); -kw("let"); -kw("const"); -kw("while", { isLoop: true }); -kw("with"); -kw("new", { beforeExpr: true, startsExpr: true }); -kw("this", startsExpr); -kw("super", startsExpr); -kw("class"); -kw("extends", beforeExpr); -kw("export"); -kw("import"); -kw("yield", { beforeExpr: true, startsExpr: true }); -kw("null", startsExpr); -kw("true", startsExpr); -kw("false", startsExpr); -kw("in", { beforeExpr: true, binop: 7 }); -kw("instanceof", { beforeExpr: true, binop: 7 }); -kw("typeof", { beforeExpr: true, prefix: true, startsExpr: true }); -kw("void", { beforeExpr: true, prefix: true, startsExpr: true }); -kw("delete", { beforeExpr: true, prefix: true, startsExpr: true });
\ No newline at end of file |