diff --git a/lib/parser.js b/lib/parser.js index 6aaccb0..eedfcef 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -1,6 +1,7 @@ "use strict"; const http = require('http'); const https = require('https'); +const zlib = require('zlib'); const xml2js = require('xml2js'); const url = require('url'); @@ -10,6 +11,7 @@ const utils = require('./utils'); const DEFAULT_HEADERS = { 'User-Agent': 'rss-parser', 'Accept': 'application/rss+xml', + 'Accept-Encoding': 'gzip, deflate, br', } const DEFAULT_MAX_REDIRECTS = 5; const DEFAULT_TIMEOUT = 60000; @@ -108,11 +110,21 @@ class Parser { } let encoding = utils.getEncodingFromContentType(res.headers['content-type']); - res.setEncoding(encoding); - res.on('data', (chunk) => { + let contentEncoding = (res.headers['content-encoding'] || '').toLowerCase(); + let stream = res; + if (contentEncoding === 'gzip' || contentEncoding === 'x-gzip') { + stream = res.pipe(zlib.createGunzip()); + } else if (contentEncoding === 'deflate' || contentEncoding === 'x-deflate') { + stream = res.pipe(zlib.createInflate()); + } else if (contentEncoding === 'br') { + stream = res.pipe(zlib.createBrotliDecompress()); + } + stream.setEncoding(encoding); + stream.on('data', (chunk) => { xml += chunk; }); - res.on('end', () => { + stream.on('error', reject); + stream.on('end', () => { return this.parseString(xml).then(resolve, reject); }); }) diff --git a/test/parser.js b/test/parser.js index 034c72f..2596614 100644 --- a/test/parser.js +++ b/test/parser.js @@ -2,6 +2,7 @@ var fs = require('fs'); var HTTP = require('http'); +var Zlib = require('zlib'); var Parser = require('../index.js'); @@ -212,6 +213,28 @@ describe('Parser', function() { }); }); + it('should parse URL that returns gzip-compressed data even without an Accept-Encoding request', function(done) { + var INPUT_FILE = __dirname + '/input/reddit.rss'; + var OUTPUT_FILE = __dirname + '/output/reddit.json'; + var server = HTTP.createServer(function(req, res) { + var gzipped = Zlib.gzipSync(fs.readFileSync(INPUT_FILE)); + res.setHeader('Content-Encoding', 'gzip'); + res.end(gzipped); + }); + server.listen(function() { + var port = server.address().port; + var url = 'http://localhost:' + port; + let parser = new Parser(); + parser.parseURL(url, function(err, parsed) { + Expect(err).to.equal(null); + var expected = JSON.parse(fs.readFileSync(OUTPUT_FILE, 'utf8')); + Expect({feed: parsed}).to.deep.equal(expected); + server.close(); + done(); + }); + }); + }); + it('should use proper encoding', function(done) { var INPUT_FILE = __dirname + '/input/encoding.rss'; var OUTPUT_FILE = __dirname + '/output/encoding.json';