fix: decompress gzip/deflate/br responses in parseURL()

lib/parser.js

@@ -1,6 +1,7 @@
 "use strict";
 const http = require('http');
 const https = require('https');
+const zlib = require('zlib');
 const xml2js = require('xml2js');
 const url = require('url');
 
@@ -10,6 +11,7 @@ const utils = require('./utils');
 const DEFAULT_HEADERS = {
   'User-Agent': 'rss-parser',
   'Accept': 'application/rss+xml',
+  'Accept-Encoding': 'gzip, deflate, br',
 }
 const DEFAULT_MAX_REDIRECTS = 5;
 const DEFAULT_TIMEOUT = 60000;
@@ -108,11 +110,21 @@ class Parser {
         }
 
         let encoding = utils.getEncodingFromContentType(res.headers['content-type']);
-        res.setEncoding(encoding);
-        res.on('data', (chunk) => {
+        let contentEncoding = (res.headers['content-encoding'] || '').toLowerCase();
+        let stream = res;
+        if (contentEncoding === 'gzip' || contentEncoding === 'x-gzip') {
+          stream = res.pipe(zlib.createGunzip());
+        } else if (contentEncoding === 'deflate' || contentEncoding === 'x-deflate') {
+          stream = res.pipe(zlib.createInflate());
+        } else if (contentEncoding === 'br') {
+          stream = res.pipe(zlib.createBrotliDecompress());
+        }
+        stream.setEncoding(encoding);
+        stream.on('data', (chunk) => {
           xml += chunk;
         });
-        res.on('end', () => {
+        stream.on('error', reject);
+        stream.on('end', () => {
           return this.parseString(xml).then(resolve, reject);
         });
       })

test/parser.js

@@ -2,6 +2,7 @@
 
 var fs = require('fs');
 var HTTP = require('http');
+var Zlib = require('zlib');
 
 var Parser = require('../index.js');
 
@@ -212,6 +213,28 @@ describe('Parser', function() {
     });
   });
 
+  it('should parse URL that returns gzip-compressed data even without an Accept-Encoding request', function(done) {
+    var INPUT_FILE = __dirname + '/input/reddit.rss';
+    var OUTPUT_FILE = __dirname + '/output/reddit.json';
+    var server = HTTP.createServer(function(req, res) {
+      var gzipped = Zlib.gzipSync(fs.readFileSync(INPUT_FILE));
+      res.setHeader('Content-Encoding', 'gzip');
+      res.end(gzipped);
+    });
+    server.listen(function() {
+      var port = server.address().port;
+      var url = 'http://localhost:' + port;
+      let parser = new Parser();
+      parser.parseURL(url, function(err, parsed) {
+        Expect(err).to.equal(null);
+        var expected = JSON.parse(fs.readFileSync(OUTPUT_FILE, 'utf8'));
+        Expect({feed: parsed}).to.deep.equal(expected);
+        server.close();
+        done();
+      });
+    });
+  });
+
   it('should use proper encoding', function(done) {
     var INPUT_FILE = __dirname + '/input/encoding.rss';
     var OUTPUT_FILE = __dirname + '/output/encoding.json';