From 363723fc84f7b8477592e0105aeb331ec9a017af Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Mon, 14 Aug 2017 05:01:11 +0200 Subject: node_modules --- node_modules/acorn/src/tokenize.js | 687 ------------------------------------- 1 file changed, 687 deletions(-) delete mode 100644 node_modules/acorn/src/tokenize.js (limited to 'node_modules/acorn/src/tokenize.js') diff --git a/node_modules/acorn/src/tokenize.js b/node_modules/acorn/src/tokenize.js deleted file mode 100644 index 917a013b8..000000000 --- a/node_modules/acorn/src/tokenize.js +++ /dev/null @@ -1,687 +0,0 @@ -import {isIdentifierStart, isIdentifierChar} from "./identifier" -import {types as tt, keywords as keywordTypes} from "./tokentype" -import {Parser} from "./state" -import {SourceLocation} from "./locutil" -import {lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace} from "./whitespace" - -// Object type used to represent tokens. Note that normally, tokens -// simply exist as properties on the parser object. This is only -// used for the onToken callback and the external tokenizer. - -export class Token { - constructor(p) { - this.type = p.type - this.value = p.value - this.start = p.start - this.end = p.end - if (p.options.locations) - this.loc = new SourceLocation(p, p.startLoc, p.endLoc) - if (p.options.ranges) - this.range = [p.start, p.end] - } -} - -// ## Tokenizer - -const pp = Parser.prototype - -// Are we running under Rhino? -const isRhino = typeof Packages == "object" && Object.prototype.toString.call(Packages) == "[object JavaPackage]" - -// Move to the next token - -pp.next = function() { - if (this.options.onToken) - this.options.onToken(new Token(this)) - - this.lastTokEnd = this.end - this.lastTokStart = this.start - this.lastTokEndLoc = this.endLoc - this.lastTokStartLoc = this.startLoc - this.nextToken() -} - -pp.getToken = function() { - this.next() - return new Token(this) -} - -// If we're in an ES6 environment, make parsers iterable -if (typeof Symbol !== "undefined") - pp[Symbol.iterator] = function() { - return { - next: () => { - let token = this.getToken() - return { - done: token.type === tt.eof, - value: token - } - } - } - } - -// Toggle strict mode. Re-reads the next number or string to please -// pedantic tests (`"use strict"; 010;` should fail). - -pp.curContext = function() { - return this.context[this.context.length - 1] -} - -// Read a single token, updating the parser object's token-related -// properties. - -pp.nextToken = function() { - let curContext = this.curContext() - if (!curContext || !curContext.preserveSpace) this.skipSpace() - - this.start = this.pos - if (this.options.locations) this.startLoc = this.curPosition() - if (this.pos >= this.input.length) return this.finishToken(tt.eof) - - if (curContext.override) return curContext.override(this) - else this.readToken(this.fullCharCodeAtPos()) -} - -pp.readToken = function(code) { - // Identifier or keyword. '\uXXXX' sequences are allowed in - // identifiers, so '\' also dispatches to that. - if (isIdentifierStart(code, this.options.ecmaVersion >= 6) || code === 92 /* '\' */) - return this.readWord() - - return this.getTokenFromCode(code) -} - -pp.fullCharCodeAtPos = function() { - let code = this.input.charCodeAt(this.pos) - if (code <= 0xd7ff || code >= 0xe000) return code - let next = this.input.charCodeAt(this.pos + 1) - return (code << 10) + next - 0x35fdc00 -} - -pp.skipBlockComment = function() { - let startLoc = this.options.onComment && this.curPosition() - let start = this.pos, end = this.input.indexOf("*/", this.pos += 2) - if (end === -1) this.raise(this.pos - 2, "Unterminated comment") - this.pos = end + 2 - if (this.options.locations) { - lineBreakG.lastIndex = start - let match - while ((match = lineBreakG.exec(this.input)) && match.index < this.pos) { - ++this.curLine - this.lineStart = match.index + match[0].length - } - } - if (this.options.onComment) - this.options.onComment(true, this.input.slice(start + 2, end), start, this.pos, - startLoc, this.curPosition()) -} - -pp.skipLineComment = function(startSkip) { - let start = this.pos - let startLoc = this.options.onComment && this.curPosition() - let ch = this.input.charCodeAt(this.pos += startSkip) - while (this.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) { - ++this.pos - ch = this.input.charCodeAt(this.pos) - } - if (this.options.onComment) - this.options.onComment(false, this.input.slice(start + startSkip, this.pos), start, this.pos, - startLoc, this.curPosition()) -} - -// Called at the start of the parse and after every token. Skips -// whitespace and comments, and. - -pp.skipSpace = function() { - loop: while (this.pos < this.input.length) { - let ch = this.input.charCodeAt(this.pos) - switch (ch) { - case 32: case 160: // ' ' - ++this.pos - break - case 13: - if (this.input.charCodeAt(this.pos + 1) === 10) { - ++this.pos - } - case 10: case 8232: case 8233: - ++this.pos - if (this.options.locations) { - ++this.curLine - this.lineStart = this.pos - } - break - case 47: // '/' - switch (this.input.charCodeAt(this.pos + 1)) { - case 42: // '*' - this.skipBlockComment() - break - case 47: - this.skipLineComment(2) - break - default: - break loop - } - break - default: - if (ch > 8 && ch < 14 || ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) { - ++this.pos - } else { - break loop - } - } - } -} - -// Called at the end of every token. Sets `end`, `val`, and -// maintains `context` and `exprAllowed`, and skips the space after -// the token, so that the next one's `start` will point at the -// right position. - -pp.finishToken = function(type, val) { - this.end = this.pos - if (this.options.locations) this.endLoc = this.curPosition() - let prevType = this.type - this.type = type - this.value = val - - this.updateContext(prevType) -} - -// ### Token reading - -// This is the function that is called to fetch the next token. It -// is somewhat obscure, because it works in character codes rather -// than characters, and because operator parsing has been inlined -// into it. -// -// All in the name of speed. -// -pp.readToken_dot = function() { - let next = this.input.charCodeAt(this.pos + 1) - if (next >= 48 && next <= 57) return this.readNumber(true) - let next2 = this.input.charCodeAt(this.pos + 2) - if (this.options.ecmaVersion >= 6 && next === 46 && next2 === 46) { // 46 = dot '.' - this.pos += 3 - return this.finishToken(tt.ellipsis) - } else { - ++this.pos - return this.finishToken(tt.dot) - } -} - -pp.readToken_slash = function() { // '/' - let next = this.input.charCodeAt(this.pos + 1) - if (this.exprAllowed) { ++this.pos; return this.readRegexp() } - if (next === 61) return this.finishOp(tt.assign, 2) - return this.finishOp(tt.slash, 1) -} - -pp.readToken_mult_modulo_exp = function(code) { // '%*' - let next = this.input.charCodeAt(this.pos + 1) - let size = 1 - let tokentype = code === 42 ? tt.star : tt.modulo - - // exponentiation operator ** and **= - if (this.options.ecmaVersion >= 7 && next === 42) { - ++size - tokentype = tt.starstar - next = this.input.charCodeAt(this.pos + 2) - } - - if (next === 61) return this.finishOp(tt.assign, size + 1) - return this.finishOp(tokentype, size) -} - -pp.readToken_pipe_amp = function(code) { // '|&' - let next = this.input.charCodeAt(this.pos + 1) - if (next === code) return this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2) - if (next === 61) return this.finishOp(tt.assign, 2) - return this.finishOp(code === 124 ? tt.bitwiseOR : tt.bitwiseAND, 1) -} - -pp.readToken_caret = function() { // '^' - let next = this.input.charCodeAt(this.pos + 1) - if (next === 61) return this.finishOp(tt.assign, 2) - return this.finishOp(tt.bitwiseXOR, 1) -} - -pp.readToken_plus_min = function(code) { // '+-' - let next = this.input.charCodeAt(this.pos + 1) - if (next === code) { - if (next == 45 && this.input.charCodeAt(this.pos + 2) == 62 && - lineBreak.test(this.input.slice(this.lastTokEnd, this.pos))) { - // A `-->` line comment - this.skipLineComment(3) - this.skipSpace() - return this.nextToken() - } - return this.finishOp(tt.incDec, 2) - } - if (next === 61) return this.finishOp(tt.assign, 2) - return this.finishOp(tt.plusMin, 1) -} - -pp.readToken_lt_gt = function(code) { // '<>' - let next = this.input.charCodeAt(this.pos + 1) - let size = 1 - if (next === code) { - size = code === 62 && this.input.charCodeAt(this.pos + 2) === 62 ? 3 : 2 - if (this.input.charCodeAt(this.pos + size) === 61) return this.finishOp(tt.assign, size + 1) - return this.finishOp(tt.bitShift, size) - } - if (next == 33 && code == 60 && this.input.charCodeAt(this.pos + 2) == 45 && - this.input.charCodeAt(this.pos + 3) == 45) { - if (this.inModule) this.unexpected() - // `