Merge pull request #666 from ryanbrandenburg/dev/rybrande/LargeJson

Handle large json failures

Author: JamieMagee

providers/process/scancode.js

@@ -54,6 +54,7 @@ class ScanCodeProcessor extends AbstractProcessor {
         maxBuffer: 5 * 1024 * 1024
       })
     } catch (error) {
+      this.logger.error(error, request.meta)
       // TODO see if the new version of ScanCode has a better way of differentiating errors
       if (this._isRealError(error) || this._hasRealErrors(file.name)) {
         request.markDead('Error', error ? error.message : 'ScanCode run failed')
@@ -91,19 +92,26 @@ class ScanCodeProcessor extends AbstractProcessor {
   // Scan the results file for any errors that are not just timeouts or other known errors
   // TODO do we need to do this anymore
   _hasRealErrors(resultFile) {
-    const results = JSON.parse(fs.readFileSync(resultFile))
-    return results.files.some(
-      file =>
-        file.scan_errors &&
-        file.scan_errors.some(error => {
-          return !(
-            error.includes('ERROR: Processing interrupted: timeout after') ||
-            error.includes('ValueError:') ||
-            error.includes('package.json') ||
-            error.includes('UnicodeDecodeError')
-          )
-        })
-    )
+    try {
+      const results = JSON.parse(fs.readFileSync(resultFile))
+      return results.files.some(
+        file =>
+          file.scan_errors &&
+          file.scan_errors.some(error => {
+            return !(
+              error.includes('ERROR: Processing interrupted: timeout after') ||
+              error.includes('ValueError:') ||
+              error.includes('package.json') ||
+              error.includes('UnicodeDecodeError')
+            )
+          })
+      )
+    } catch (e) {
+      // This might happen if the results file is >512mb, but regardless our error handling should not fail.
+      this.logger.error(`Could not parse ScanCode results file ${resultFile}: ${e.message}`)
+      // Assume it's broken if we could not parse it
+      return true
+    }
   }
 
   _detectVersion() {

test/unit/providers/process/scancodeTests.js

@@ -39,7 +39,7 @@ describe('ScanCode misc', () => {
     const handlerFactory = proxyquire('../../../../providers/process/scancode', {
       fs: fsStub
     })
-    Handler = handlerFactory({ logger: { log: () => {} } })
+    Handler = handlerFactory({ logger: { log: () => {}, error: () => {} } })
     Handler._resultBox = resultBox
   })
 
@@ -108,7 +108,7 @@ function setup(fixture, error, versionError) {
     timeout: 200,
     processes: 2,
     format: 'json',
-    logger: { log: sinon.stub(), info: sinon.stub() }
+    logger: { log: sinon.stub(), info: sinon.stub(), error: sinon.stub() }
   }
   const testRequest = new request('npm', 'cd:/npm/npmjs/-/test/1.1')
   testRequest.document = { _metadata: { links: {} }, location: '/test' }