From abd94a7f5a50f43c797a11b53549ae48fff667c3 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Mon, 10 Oct 2016 03:43:44 +0200 Subject: add node_modules to address #4364 --- node_modules/gettext-parser/lib/mocompiler.js | 237 ++++++++++++ node_modules/gettext-parser/lib/moparser.js | 202 ++++++++++ node_modules/gettext-parser/lib/pocompiler.js | 225 +++++++++++ node_modules/gettext-parser/lib/poparser.js | 525 ++++++++++++++++++++++++++ node_modules/gettext-parser/lib/shared.js | 120 ++++++ 5 files changed, 1309 insertions(+) create mode 100644 node_modules/gettext-parser/lib/mocompiler.js create mode 100644 node_modules/gettext-parser/lib/moparser.js create mode 100644 node_modules/gettext-parser/lib/pocompiler.js create mode 100644 node_modules/gettext-parser/lib/poparser.js create mode 100644 node_modules/gettext-parser/lib/shared.js (limited to 'node_modules/gettext-parser/lib') diff --git a/node_modules/gettext-parser/lib/mocompiler.js b/node_modules/gettext-parser/lib/mocompiler.js new file mode 100644 index 000000000..0e848c7fd --- /dev/null +++ b/node_modules/gettext-parser/lib/mocompiler.js @@ -0,0 +1,237 @@ +'use strict'; + +var encoding = require('encoding'); +var sharedFuncs = require('./shared'); + +/** + * Exposes general compiler function. Takes a translation + * object as a parameter and returns binary MO object + * + * @param {Object} table Translation object + * @return {Buffer} Compiled binary MO object + */ +module.exports = function(table) { + var compiler = new Compiler(table); + return compiler.compile(); +}; + +/** + * Creates a MO compiler object. + * + * @constructor + * @param {Object} table Translation table as defined in the README + */ +function Compiler(table) { + this._table = table || {}; + this._table.headers = this._table.headers || {}; + this._table.translations = this._table.translations || {}; + + this._translations = []; + + this._writeFunc = 'writeUInt32LE'; + + this._handleCharset(); +} + +/** + * Magic bytes for the generated binary data + */ +Compiler.prototype.MAGIC = 0x950412de; + +/** + * Handles header values, replaces or adds (if needed) a charset property + */ +Compiler.prototype._handleCharset = function() { + var parts = (this._table.headers['content-type'] || 'text/plain').split(';'), + contentType = parts.shift(), + charset = sharedFuncs.formatCharset(this._table.charset), + params = []; + + params = parts.map(function(part) { + var parts = part.split('='), + key = parts.shift().trim(), + value = parts.join('='); + + if (key.toLowerCase() === 'charset') { + if (!charset) { + charset = sharedFuncs.formatCharset(value.trim() || 'utf-8'); + } + return 'charset=' + charset; + } + + return part; + }); + + if (!charset) { + charset = this._table.charset || 'utf-8'; + params.push('charset=' + charset); + } + + this._table.charset = charset; + this._table.headers['content-type'] = contentType + '; ' + params.join('; '); + + this._charset = charset; +}; + +/** + * Generates an array of translation strings + * in the form of [{msgid:... , msgstr:...}] + * + * @return {Array} Translation strings array + */ +Compiler.prototype._generateList = function() { + var list = []; + + list.push({ + msgid: new Buffer(0), + msgstr: encoding.convert(sharedFuncs.generateHeader(this._table.headers), this._charset) + }); + + Object.keys(this._table.translations).forEach((function(msgctxt) { + if (typeof this._table.translations[msgctxt] !== 'object') { + return; + } + Object.keys(this._table.translations[msgctxt]).forEach((function(msgid) { + if (typeof this._table.translations[msgctxt][msgid] !== 'object') { + return; + } + if (msgctxt === '' && msgid === '') { + return; + } + + var msgid_plural = this._table.translations[msgctxt][msgid].msgid_plural, + key = msgid, + value; + + if (msgctxt) { + key = msgctxt + '\u0004' + key; + } + + if (msgid_plural) { + key += '\u0000' + msgid_plural; + } + + value = [].concat(this._table.translations[msgctxt][msgid].msgstr || []).join('\u0000'); + + list.push({ + msgid: encoding.convert(key, this._charset), + msgstr: encoding.convert(value, this._charset) + }); + }).bind(this)); + }).bind(this)); + + return list; +}; + +/** + * Calculate buffer size for the final binary object + * + * @param {Array} list An array of translation strings from _generateList + * @return {Object} Size data of {msgid, msgstr, total} + */ +Compiler.prototype._calculateSize = function(list) { + var msgidLength = 0, + msgstrLength = 0, + totalLength = 0; + + list.forEach(function(translation) { + msgidLength += translation.msgid.length + 1; // + extra 0x00 + msgstrLength += translation.msgstr.length + 1; // + extra 0x00 + }); + + totalLength = 4 + // magic number + 4 + // revision + 4 + // string count + 4 + // original string table offset + 4 + // translation string table offset + 4 + // hash table size + 4 + // hash table offset + (4 + 4) * list.length + // original string table + (4 + 4) * list.length + // translations string table + msgidLength + // originals + msgstrLength; // translations + + return { + msgid: msgidLength, + msgstr: msgstrLength, + total: totalLength + }; +}; + +/** + * Generates the binary MO object from the translation list + * + * @param {Array} list translation list + * @param {Object} size Byte size information + * @return {Buffer} Compiled MO object + */ +Compiler.prototype._build = function(list, size) { + var returnBuffer = new Buffer(size.total), + curPosition = 0, + i, len; + + // magic + returnBuffer[this._writeFunc](this.MAGIC, 0); + + // revision + returnBuffer[this._writeFunc](0, 4); + + // string count + returnBuffer[this._writeFunc](list.length, 8); + + // original string table offset + returnBuffer[this._writeFunc](28, 12); + + // translation string table offset + returnBuffer[this._writeFunc](28 + (4 + 4) * list.length, 16); + + // hash table size + returnBuffer[this._writeFunc](0, 20); + + // hash table offset + returnBuffer[this._writeFunc](28 + (4 + 4) * list.length, 24); + + // build originals table + curPosition = 28 + 2 * (4 + 4) * list.length; + for (i = 0, len = list.length; i < len; i++) { + list[i].msgid.copy(returnBuffer, curPosition); + returnBuffer[this._writeFunc](list[i].msgid.length, 28 + i * 8); + returnBuffer[this._writeFunc](curPosition, 28 + i * 8 + 4); + returnBuffer[curPosition + list[i].msgid.length] = 0x00; + curPosition += list[i].msgid.length + 1; + } + + // build translations table + for (i = 0, len = list.length; i < len; i++) { + list[i].msgstr.copy(returnBuffer, curPosition); + returnBuffer[this._writeFunc](list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8); + returnBuffer[this._writeFunc](curPosition, 28 + (4 + 4) * list.length + i * 8 + 4); + returnBuffer[curPosition + list[i].msgstr.length] = 0x00; + curPosition += list[i].msgstr.length + 1; + } + + return returnBuffer; +}; + +/** + * Compiles translation object into a binary MO object + * + * @return {Buffer} Compiled MO object + */ +Compiler.prototype.compile = function() { + var list = this._generateList(), + size = this._calculateSize(list); + + // sort by msgid + list.sort(function(a, b) { + if (a.msgid > b.msgid) { + return 1; + } + if (a.msgid < b.msgid) { + return -1; + } + return 0; + }); + + return this._build(list, size); +}; \ No newline at end of file diff --git a/node_modules/gettext-parser/lib/moparser.js b/node_modules/gettext-parser/lib/moparser.js new file mode 100644 index 000000000..8c204716b --- /dev/null +++ b/node_modules/gettext-parser/lib/moparser.js @@ -0,0 +1,202 @@ +'use strict'; + +var encoding = require('encoding'); +var sharedFuncs = require('./shared'); + +/** + * Parses a binary MO object into translation table + * + * @param {Buffer} buffer Binary MO object + * @param {String} [defaultCharset] Default charset to use + * @return {Object} Translation object + */ +module.exports = function(buffer, defaultCharset) { + var parser = new Parser(buffer, defaultCharset); + return parser.parse(); +}; + +/** + * Creates a MO parser object. + * + * @constructor + * @param {Buffer} fileContents Binary MO object + * @param {String} [defaultCharset] Default charset to use + */ +function Parser(fileContents, defaultCharset) { + + this._fileContents = fileContents; + + /** + * Method name for writing int32 values, default littleendian + */ + this._writeFunc = 'writeUInt32LE'; + + /** + * Method name for reading int32 values, default littleendian + */ + this._readFunc = 'readUInt32LE'; + + this._charset = defaultCharset || 'iso-8859-1'; + + this._table = { + charset: this._charset, + headers: undefined, + translations: {} + }; +} + +/** + * Magic constant to check the endianness of the input file + */ +Parser.prototype.MAGIC = 0x950412de; + +/** + * Checks if number values in the input file are in big- or littleendian format. + * + * @return {Boolean} Return true if magic was detected + */ +Parser.prototype._checkMagick = function() { + if (this._fileContents.readUInt32LE(0) === this.MAGIC) { + this._readFunc = 'readUInt32LE'; + this._writeFunc = 'writeUInt32LE'; + return true; + } else if (this._fileContents.readUInt32BE(0) === this.MAGIC) { + this._readFunc = 'readUInt32BE'; + this._writeFunc = 'writeUInt32BE'; + return true; + } else { + return false; + } +}; + +/** + * Read the original strings and translations from the input MO file. Use the + * first translation string in the file as the header. + */ +Parser.prototype._loadTranslationTable = function() { + var offsetOriginals = this._offsetOriginals, + offsetTranslations = this._offsetTranslations, + position, length, + msgid, msgstr; + + for (var i = 0; i < this._total; i++) { + // msgid string + length = this._fileContents[this._readFunc](offsetOriginals); + offsetOriginals += 4; + position = this._fileContents[this._readFunc](offsetOriginals); + offsetOriginals += 4; + msgid = this._fileContents.slice(position, position + length); + + // matching msgstr + length = this._fileContents[this._readFunc](offsetTranslations); + offsetTranslations += 4; + position = this._fileContents[this._readFunc](offsetTranslations); + offsetTranslations += 4; + msgstr = this._fileContents.slice(position, position + length); + + if (!i && !msgid.toString()) { + this._handleCharset(msgstr); + } + + msgid = encoding.convert(msgid, 'utf-8', this._charset).toString('utf-8'); + msgstr = encoding.convert(msgstr, 'utf-8', this._charset).toString('utf-8'); + + this._addString(msgid, msgstr); + } + + // dump the file contents object + this._fileContents = null; +}; + +/** + * Detects charset for MO strings from the header + * + * @param {Buffer} headers Header value + */ +Parser.prototype._handleCharset = function(headers) { + + var headersStr = headers.toString(), + match; + + if ((match = headersStr.match(/[; ]charset\s*=\s*([\w\-]+)/i))) { + this._charset = this._table.charset = sharedFuncs.formatCharset(match[1], this._charset); + } + + headers = encoding.convert(headers, 'utf-8', this._charset).toString('utf-8'); + + this._table.headers = sharedFuncs.parseHeader(headers); +}; + +/** + * Adds a translation to the translation object + * + * @param {String} msgid Original string + * @params {String} msgstr Translation for the original string + */ +Parser.prototype._addString = function(msgid, msgstr) { + var translation = {}, + parts, msgctxt, msgid_plural; + + msgid = msgid.split('\u0004'); + if (msgid.length > 1) { + msgctxt = msgid.shift(); + translation.msgctxt = msgctxt; + } else { + msgctxt = ''; + } + msgid = msgid.join('\u0004'); + + parts = msgid.split('\u0000'); + msgid = parts.shift(); + + translation.msgid = msgid; + + if ((msgid_plural = parts.join('\u0000'))) { + translation.msgid_plural = msgid_plural; + } + + msgstr = msgstr.split('\u0000'); + translation.msgstr = [].concat(msgstr || []); + + if (!this._table.translations[msgctxt]) { + this._table.translations[msgctxt] = {}; + } + + this._table.translations[msgctxt][msgid] = translation; +}; + +/** + * Parses the MO object and returns translation table + * + * @return {Object} Translation table + */ +Parser.prototype.parse = function() { + if (!this._checkMagick()) { + return false; + } + + /** + * GetText revision nr, usually 0 + */ + this._revision = this._fileContents[this._readFunc](4); + + /** + * Total count of translated strings + */ + this._total = this._fileContents[this._readFunc](8); + + /** + * Offset position for original strings table + */ + this._offsetOriginals = this._fileContents[this._readFunc](12); + + /** + * Offset position for translation strings table + */ + this._offsetTranslations = this._fileContents[this._readFunc](16); + + // Load translations into this._translationTable + this._loadTranslationTable(); + + return this._table; +}; \ No newline at end of file diff --git a/node_modules/gettext-parser/lib/pocompiler.js b/node_modules/gettext-parser/lib/pocompiler.js new file mode 100644 index 000000000..0f3c1a166 --- /dev/null +++ b/node_modules/gettext-parser/lib/pocompiler.js @@ -0,0 +1,225 @@ +'use strict'; + +var encoding = require('encoding'); +var sharedFuncs = require('./shared'); + +/** + * Exposes general compiler function. Takes a translation + * object as a parameter and returns PO object + * + * @param {Object} table Translation object + * @return {Buffer} Compiled PO object + */ +module.exports = function(table) { + var compiler = new Compiler(table); + return compiler.compile(); +}; + +/** + * Creates a PO compiler object. + * + * @constructor + * @param {Object} table Translation table to be compiled + */ +function Compiler(table) { + this._table = table || {}; + this._table.headers = this._table.headers || {}; + this._table.translations = this._table.translations || {}; + this._translations = []; + this._handleCharset(); +} + +/** + * Converts a comments object to a comment string. The comment object is + * in the form of {translator:'', reference: '', extracted: '', flag: '', previous:''} + * + * @param {Object} comments A comments object + * @return {String} A comment string for the PO file + */ +Compiler.prototype._drawComments = function(comments) { + var lines = []; + var types = [{ + key: 'translator', + prefix: '# ' + }, { + key: 'reference', + prefix: '#: ' + }, { + key: 'extracted', + prefix: '#. ' + }, { + key: 'flag', + prefix: '#, ' + }, { + key: 'previous', + prefix: '#| ' + }]; + + types.forEach(function(type) { + if (!comments[type.key]) { + return; + } + comments[type.key].split(/\r?\n|\r/).forEach(function(line) { + lines.push(type.prefix + line); + }); + }); + + return lines.join('\n'); +}; + +/** + * Builds a PO string for a single translation object + * + * @param {Object} block Translation object + * @param {Object} [override] Properties of this object will override `block` properties + * @return {String} Translation string for a single object + */ +Compiler.prototype._drawBlock = function(block, override) { + + override = override || {}; + + var response = [], + comments = override.comments || block.comments, + msgctxt = override.msgctxt || block.msgctxt, + msgid = override.msgid || block.msgid, + msgid_plural = override.msgid_plural || block.msgid_plural, + msgstr = [].concat(override.msgstr || block.msgstr); + + + // add comments + if (comments && (comments = this._drawComments(comments))) { + response.push(comments); + } + + if (msgctxt) { + response.push(this._addPOString('msgctxt', msgctxt)); + } + + response.push(this._addPOString('msgid', msgid || '')); + + if (msgid_plural) { + response.push(this._addPOString('msgid_plural', msgid_plural)); + } + + if (msgstr.length <= 1) { + response.push(this._addPOString('msgstr', msgstr[0] || '')); + } else { + msgstr.forEach((function(msgstr, i) { + response.push(this._addPOString('msgstr[' + i + ']', msgstr || '')); + }).bind(this)); + } + + return response.join('\n'); +}; + +/** + * Escapes and joins a key and a value for the PO string + * + * @param {String} key Key name + * @param {String} value Key value + * @return {String} Joined and escaped key-value pair + */ +Compiler.prototype._addPOString = function(key, value) { + var line; + + key = (key || '').toString(); + + // escape newlines and quotes + value = (value || '').toString(). + replace(/\\/g, '\\\\'). + replace(/"/g, '\\"'). + replace(/\t/g, '\\t'). + replace(/\r/g, '\\r'). + replace(/\n/g, '\\n'); + + var lines = sharedFuncs.foldLine(value); + + if (lines.length < 2) { + return key + ' "' + (lines.shift() || '') + '"'; + } else { + return key + ' ""\n"' + lines.join('"\n"') + '"'; + } + + if (value.match(/\n/)) { + value = value.replace(/\n/g, '\\n\n').replace(/\n$/, ''); + line = ('\n' + value).split('\n').map(function(l) { + return '"' + l + '"'; + }).join('\n'); + } else { + line = '"' + value + '"'; + } + + return key + ' ' + line; +}; + +/** + * Handles header values, replaces or adds (if needed) a charset property + */ +Compiler.prototype._handleCharset = function() { + var parts = (this._table.headers['content-type'] || 'text/plain').split(';'); + var contentType = parts.shift(); + var charset = sharedFuncs.formatCharset(this._table.charset); + var params = []; + + params = parts.map(function(part) { + var parts = part.split('='), + key = parts.shift().trim(), + value = parts.join('='); + + if (key.toLowerCase() === 'charset') { + if (!charset) { + charset = sharedFuncs.formatCharset(value.trim() || 'utf-8'); + } + return 'charset=' + charset; + } + + return part; + }); + + if (!charset) { + charset = this._table.charset || 'utf-8'; + params.push('charset=' + charset); + } + + this._table.charset = charset; + this._table.headers['content-type'] = contentType + '; ' + params.join('; '); + + this._charset = charset; +}; + +/** + * Compiles translation object into a PO object + * + * @return {Buffer} Compiled PO object + */ +Compiler.prototype.compile = function() { + + var response = [], + headerBlock = this._table.translations[''] && this._table.translations[''][''] || {}; + + response.push(this._drawBlock(headerBlock, { + msgstr: sharedFuncs.generateHeader(this._table.headers) + })); + + Object.keys(this._table.translations).forEach((function(msgctxt) { + if (typeof this._table.translations[msgctxt] !== 'object') { + return; + } + Object.keys(this._table.translations[msgctxt]).forEach((function(msgid) { + if (typeof this._table.translations[msgctxt][msgid] !== 'object') { + return; + } + if (msgctxt === '' && msgid === '') { + return; + } + + response.push(this._drawBlock(this._table.translations[msgctxt][msgid])); + }).bind(this)); + }).bind(this)); + + if (this._charset === 'utf-8' || this._charset === 'ascii') { + return new Buffer(response.join('\n\n'), 'utf-8'); + } else { + return encoding.convert(response.join('\n\n'), this._charset); + } +}; \ No newline at end of file diff --git a/node_modules/gettext-parser/lib/poparser.js b/node_modules/gettext-parser/lib/poparser.js new file mode 100644 index 000000000..e215bca08 --- /dev/null +++ b/node_modules/gettext-parser/lib/poparser.js @@ -0,0 +1,525 @@ +'use strict'; + +var encoding = require('encoding'); +var sharedFuncs = require('./shared'); +var Transform = require('stream').Transform; +var util = require('util'); + +/** + * Parses a PO object into translation table + * + * @param {Buffer|String} buffer PO object + * @param {String} [defaultCharset] Default charset to use + * @return {Object} Translation object + */ +module.exports.parse = function(buffer, defaultCharset) { + var parser = new Parser(buffer, defaultCharset); + return parser.parse(); +}; + +/** + * Parses a PO stream, emits translation table in object mode + * + * @param {String} [defaultCharset] Default charset to use + * @param {String} [options] Stream options + * @return {Stream} Transform stream + */ +module.exports.stream = function(defaultCharset, options) { + return new PoParserTransform(defaultCharset, options); +}; + +/** + * Creates a PO parser object. If PO object is a string, + * UTF-8 will be used as the charset + * + * @constructor + * @param {Buffer|String} fileContents PO object + * @param {String} [defaultCharset] Default charset to use + */ +function Parser(fileContents, defaultCharset) { + + this._charset = defaultCharset || 'iso-8859-1'; + + this._lex = []; + this._escaped = false; + this._node; + this._state = this.states.none; + + if (typeof fileContents === 'string') { + this._charset = 'utf-8'; + this._fileContents = fileContents; + } else { + this._handleCharset(fileContents); + } +} + +/** + * Parses the PO object and returns translation table + * + * @return {Object} Translation table + */ +Parser.prototype.parse = function() { + this._lexer(this._fileContents); + return this._finalize(this._lex); +}; + +/** + * Detects charset for PO strings from the header + * + * @param {Buffer} headers Header value + */ +Parser.prototype._handleCharset = function(buf) { + var str = (buf || '').toString(), + pos, headers = '', + match; + + if ((pos = str.search(/^\s*msgid/im)) >= 0) { + if ((pos = pos + str.substr(pos + 5).search(/^\s*(msgid|msgctxt)/im))) { + headers = str.substr(0, pos); + } + } + + if ((match = headers.match(/[; ]charset\s*=\s*([\w\-]+)(?:[\s;]|\\n)*"\s*$/mi))) { + this._charset = sharedFuncs.formatCharset(match[1], this._charset); + } + + if (this._charset === 'utf-8') { + this._fileContents = str; + } else { + this._fileContents = this._toString(buf); + } +}; + +Parser.prototype._toString = function(buf) { + return encoding.convert(buf, 'utf-8', this._charset).toString('utf-8'); +}; + +/** + * State constants for parsing FSM + */ +Parser.prototype.states = { + none: 0x01, + comments: 0x02, + key: 0x03, + string: 0x04 +}; + +/** + * Value types for lexer + */ +Parser.prototype.types = { + comments: 0x01, + key: 0x02, + string: 0x03 +}; + +/** + * String matches for lexer + */ +Parser.prototype.symbols = { + quotes: /["']/, + comments: /\#/, + whitespace: /\s/, + key: /[\w\-\[\]]/ +}; + +/** + * Token parser. Parsed state can be found from this._lex + * + * @param {String} chunk String + */ +Parser.prototype._lexer = function(chunk) { + var chr; + + for (var i = 0, len = chunk.length; i < len; i++) { + chr = chunk.charAt(i); + switch (this._state) { + case this.states.none: + if (chr.match(this.symbols.quotes)) { + this._node = { + type: this.types.string, + value: '', + quote: chr + }; + this._lex.push(this._node); + this._state = this.states.string; + } else if (chr.match(this.symbols.comments)) { + this._node = { + type: this.types.comments, + value: '' + }; + this._lex.push(this._node); + this._state = this.states.comments; + } else if (!chr.match(this.symbols.whitespace)) { + this._node = { + type: this.types.key, + value: chr + }; + this._lex.push(this._node); + this._state = this.states.key; + } + break; + case this.states.comments: + if (chr === '\n') { + this._state = this.states.none; + } else if (chr !== '\r') { + this._node.value += chr; + } + break; + case this.states.string: + if (this._escaped) { + switch (chr) { + case 't': + this._node.value += '\t'; + break; + case 'n': + this._node.value += '\n'; + break; + case 'r': + this._node.value += '\r'; + break; + default: + this._node.value += chr; + } + this._escaped = false; + } else { + if (chr === this._node.quote) { + this._state = this.states.none; + } else if (chr === '\\') { + this._escaped = true; + break; + } else { + this._node.value += chr; + } + this._escaped = false; + } + break; + case this.states.key: + if (!chr.match(this.symbols.key)) { + this._state = this.states.none; + i--; + } else { + this._node.value += chr; + } + break; + } + } +}; + +/** + * Join multi line strings + * + * @param {Object} tokens Parsed tokens + * @return {Object} Parsed tokens, with multi line strings joined into one + */ +Parser.prototype._joinStringValues = function(tokens) { + var lastNode, response = []; + + for (var i = 0, len = tokens.length; i < len; i++) { + if (lastNode && tokens[i].type === this.types.string && lastNode.type === this.types.string) { + lastNode.value += tokens[i].value; + } else if (lastNode && tokens[i].type === this.types.comments && lastNode.type === this.types.comments) { + lastNode.value += '\n' + tokens[i].value; + } else { + response.push(tokens[i]); + lastNode = tokens[i]; + } + } + + return response; +}; + +/** + * Parse comments into separate comment blocks + * + * @param {Object} tokens Parsed tokens + */ +Parser.prototype._parseComments = function(tokens) { + // parse comments + tokens.forEach((function(node) { + var comment, lines; + + if (node && node.type === this.types.comments) { + comment = { + translator: [], + extracted: [], + reference: [], + flag: [], + previous: [] + }; + lines = (node.value || '').split(/\n/); + lines.forEach(function(line) { + switch (line.charAt(0) || '') { + case ':': + comment.reference.push(line.substr(1).trim()); + break; + case '.': + comment.extracted.push(line.substr(1).replace(/^\s+/, '')); + break; + case ',': + comment.flag.push(line.substr(1).replace(/^\s+/, '')); + break; + case '|': + comment.previous.push(line.substr(1).replace(/^\s+/, '')); + break; + default: + comment.translator.push(line.replace(/^\s+/, '')); + } + }); + + node.value = {}; + + Object.keys(comment).forEach(function(key) { + if (comment[key] && comment[key].length) { + node.value[key] = comment[key].join('\n'); + } + }); + } + }).bind(this)); +}; + +/** + * Join gettext keys with values + * + * @param {Object} tokens Parsed tokens + * @return {Object} Tokens + */ +Parser.prototype._handleKeys = function(tokens) { + var response = [], + lastNode; + + for (var i = 0, len = tokens.length; i < len; i++) { + if (tokens[i].type === this.types.key) { + lastNode = { + key: tokens[i].value + }; + if (i && tokens[i - 1].type === this.types.comments) { + lastNode.comments = tokens[i - 1].value; + } + lastNode.value = ''; + response.push(lastNode); + } else if (tokens[i].type === this.types.string && lastNode) { + lastNode.value += tokens[i].value; + } + } + + return response; +}; + +/** + * Separate different values into individual translation objects + * + * @param {Object} tokens Parsed tokens + * @return {Object} Tokens + */ +Parser.prototype._handleValues = function(tokens) { + var response = [], + lastNode, curContext, curComments; + + for (var i = 0, len = tokens.length; i < len; i++) { + if (tokens[i].key.toLowerCase() === 'msgctxt') { + curContext = tokens[i].value; + curComments = tokens[i].comments; + } else if (tokens[i].key.toLowerCase() === 'msgid') { + lastNode = { + msgid: tokens[i].value + }; + + if (curContext) { + lastNode.msgctxt = curContext; + } + + if (curComments) { + lastNode.comments = curComments; + } + + if (tokens[i].comments && !lastNode.comments) { + lastNode.comments = tokens[i].comments; + } + + curContext = false; + curComments = false; + response.push(lastNode); + } else if (tokens[i].key.toLowerCase() === 'msgid_plural') { + if (lastNode) { + lastNode.msgid_plural = tokens[i].value; + } + + if (tokens[i].comments && !lastNode.comments) { + lastNode.comments = tokens[i].comments; + } + + curContext = false; + curComments = false; + } else if (tokens[i].key.substr(0, 6).toLowerCase() === 'msgstr') { + if (lastNode) { + lastNode.msgstr = (lastNode.msgstr || []).concat(tokens[i].value); + } + + if (tokens[i].comments && !lastNode.comments) { + lastNode.comments = tokens[i].comments; + } + + curContext = false; + curComments = false; + } + } + + return response; +}; + +/** + * Compose a translation table from tokens object + * + * @param {Object} tokens Parsed tokens + * @return {Object} Translation table + */ +Parser.prototype._normalize = function(tokens) { + var msgctxt, + table = { + charset: this._charset, + headers: undefined, + translations: {} + }; + + for (var i = 0, len = tokens.length; i < len; i++) { + msgctxt = tokens[i].msgctxt || ''; + + if (!table.translations[msgctxt]) { + table.translations[msgctxt] = {}; + } + + if (!table.headers && !msgctxt && !tokens[i].msgid) { + table.headers = sharedFuncs.parseHeader(tokens[i].msgstr[0]); + } + + table.translations[msgctxt][tokens[i].msgid] = tokens[i]; + } + + return table; +}; + +/** + * Converts parsed tokens to a translation table + * + * @param {Object} tokens Parsed tokens + * @returns {Object} Translation table + */ +Parser.prototype._finalize = function(tokens) { + var data = this._joinStringValues(tokens); + this._parseComments(data); + data = this._handleKeys(data); + data = this._handleValues(data); + + return this._normalize(data); +}; + +/** + * Creates a transform stream for parsing PO input + * + * @constructor + * @param {String} [defaultCharset] Default charset to use + * @param {String} [options] Stream options + */ +function PoParserTransform(defaultCharset, options) { + if (!options && defaultCharset && typeof defaultCharset === 'object') { + options = defaultCharset; + defaultCharset = undefined; + } + + this.defaultCharset = defaultCharset; + this._parser = false; + this._tokens = {}; + + this._cache = []; + this._cacheSize = 0; + + this.initialTreshold = options.initialTreshold || 2 * 1024; + + Transform.call(this, options); + this._writableState.objectMode = false; + this._readableState.objectMode = true; +} +util.inherits(PoParserTransform, Transform); + +/** + * Processes a chunk of the input stream + */ +PoParserTransform.prototype._transform = function(chunk, encoding, done) { + var i, len = 0; + + if (!chunk || !chunk.length) { + return done(); + } + + if (!this._parser) { + this._cache.push(chunk); + this._cacheSize += chunk.length; + + // wait until the first 1kb before parsing headers for charset + if (this._cacheSize < this.initialTreshold) { + return setImmediate(done); + } else if (this._cacheSize) { + chunk = Buffer.concat(this._cache, this._cacheSize); + this._cacheSize = 0; + this._cache = []; + } + + this._parser = new Parser(chunk, this.defaultCharset); + } else if (this._cacheSize) { + // this only happens if we had an uncompleted 8bit sequence from the last iteration + this._cache.push(chunk); + this._cacheSize += chunk.length; + chunk = Buffer.concat(this._cache, this._cacheSize); + this._cacheSize = 0; + this._cache = []; + } + + // cache 8bit bytes from the end of the chunk + // helps if the chunk ends in the middle of an utf-8 sequence + for (i = chunk.length - 1; i >= 0; i--) { + if (chunk[i] >= 0x80) { + len++; + continue; + } + break; + } + // it seems we found some 8bit bytes from the end of the string, so let's cache these + if (len) { + this._cache = [chunk.slice(chunk.length - len)]; + this._cacheSize = this._cache[0].length; + chunk = chunk.slice(0, chunk.length - len); + } + + // chunk might be empty if it only contined of 8bit bytes and these were all cached + if (chunk.length) { + this._parser._lexer(this._parser._toString(chunk)); + } + + setImmediate(done); +}; + +/** + * Once all input has been processed emit the parsed translation table as an object + */ +PoParserTransform.prototype._flush = function(done) { + var chunk; + + if (this._cacheSize) { + chunk = Buffer.concat(this._cache, this._cacheSize); + } + + if (!this._parser && chunk) { + this._parser = new Parser(chunk, this.defaultCharset); + } + + if (chunk) { + this._parser._lexer(this._parser._toString(chunk)); + } + + if (this._parser) { + this.push(this._parser._finalize(this._parser._lex)); + } + + setImmediate(done); +}; \ No newline at end of file diff --git a/node_modules/gettext-parser/lib/shared.js b/node_modules/gettext-parser/lib/shared.js new file mode 100644 index 000000000..44bfb86f7 --- /dev/null +++ b/node_modules/gettext-parser/lib/shared.js @@ -0,0 +1,120 @@ +'use strict'; + +// Expose to the world +module.exports.parseHeader = parseHeader; +module.exports.generateHeader = generateHeader; +module.exports.formatCharset = formatCharset; +module.exports.foldLine = foldLine; + +/** + * Parses a header string into an object of key-value pairs + * + * @param {String} str Header string + * @return {Object} An object of key-value pairs + */ +function parseHeader(str) { + var lines = (str || '').split('\n'), + headers = {}; + + lines.forEach(function(line) { + var parts = line.trim().split(':'), + key = (parts.shift() || '').trim().toLowerCase(), + value = parts.join(':').trim(); + if (!key) { + return; + } + headers[key] = value; + }); + + return headers; +} + +/** + * Convert first letters after - to uppercase, other lowercase + * + * @param {String} str String to be updated + * @return {String} A string with uppercase words + */ +function upperCaseWords(str) { + return (str || '').toLowerCase().trim().replace(/^(MIME|POT?(?=\-)|[a-z])|\-[a-z]/gi, function(str) { + return str.toUpperCase(); + }); +} + +/** + * Joins a header object of key value pairs into a header string + * + * @param {Object} header Object of key value pairs + * @return {String} Header string + */ +function generateHeader(header) { + var lines = []; + + Object.keys(header || {}).forEach(function(key) { + if (key) { + lines.push(upperCaseWords(key) + ': ' + (header[key] || '').trim()); + } + }); + + return lines.join('\n') + (lines.length ? '\n' : ''); +} + +/** + * Normalizes charset name. Converts utf8 to utf-8, WIN1257 to windows-1257 etc. + * + * @param {String} charset Charset name + * @return {String} Normalized charset name + */ +function formatCharset(charset, defaultCharset) { + return (charset || 'iso-8859-1').toString().toLowerCase(). + replace(/^utf[\-_]?(\d+)$/, 'utf-$1'). + replace(/^win(?:dows)?[\-_]?(\d+)$/, 'windows-$1'). + replace(/^latin[\-_]?(\d+)$/, 'iso-8859-$1'). + replace(/^(us[\-_]?)?ascii$/, 'ascii'). + replace(/^charset$/, defaultCharset || 'iso-8859-1'). + trim(); +} + +/** + * Folds long lines according to PO format + * + * @param {String} str PO formatted string to be folded + * @param {Number} [maxLen=76] Maximum allowed length for folded lines + * @return {Array} An array of lines + */ +function foldLine(str, maxLen) { + + maxLen = maxLen || 76; + + var lines = [], + curLine = '', + pos = 0, + len = str.length, + match; + + while (pos < len) { + curLine = str.substr(pos, maxLen); + + // ensure that the line never ends with a partial escaping + // make longer lines if needed + while (curLine.substr(-1) === '\\' && pos + curLine.length < len) { + curLine += str.charAt(pos + curLine.length); + } + + // ensure that if possible, line breaks are done at reasonable places + if ((match = curLine.match(/\\n/))) { + curLine = curLine.substr(0, match.index + 2); + } else if (pos + curLine.length < len) { + if ((match = curLine.match(/(\s+)[^\s]*$/)) && match.index > 0) { + curLine = curLine.substr(0, match.index + match[1].length); + } else if ((match = curLine.match(/([\x21-\x40\x5b-\x60\x7b-\x7e]+)[^\x21-\x40\x5b-\x60\x7b-\x7e]*$/)) && match.index > 0) { + curLine = curLine.substr(0, match.index + match[1].length); + } + } + + lines.push(curLine); + pos += curLine.length; + } + + return lines; +} \ No newline at end of file -- cgit v1.2.3