CodeceptJS/test/runner/run_rerun_test.js at abfff96893805bc2369988c27020c641983b1f03 · codeceptjs/CodeceptJS · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import * as chai from 'chai';
chai.should();
import { expect } from 'expect';
import { describe } from 'mocha';
import path from 'path';
import { exec } from 'child_process';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const runner = path.join(__dirname, '/../../bin/codecept.js')
const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/run-rerun/')
const codecept_run = `${runner} run-rerun`
const codecept_run_config = (config, grep) => `${codecept_run} --config ${codecept_dir}/${config} --grep "${grep || ''}"`

describe('run-rerun command', () => {
  before(() => {
    process.chdir(codecept_dir)
  })

  it('should display count of attemps', done => {
    exec(`${codecept_run_config('codecept.conf.js')} --debug`, (err, stdout) => {
      const runs = stdout.split('Run Rerun - Command --')
      // check first run
      expect(runs[1]).toContain('OK  | 1 passed')
      // expect(runs[1]).toContain('✔ OK')

      // check second run
      expect(runs[2]).toContain('OK  | 1 passed')
      // expect(runs[2]).toContain('✔ OK')

      // check third run
      expect(runs[2]).toContain('OK  | 1 passed')
      // expect(runs[2]).toContain('✔ OK')

      expect(stdout).toContain('Process run 1 of max 3, success runs 1/3')
      expect(stdout).toContain('Process run 2 of max 3, success runs 2/3')
      expect(stdout).toContain('Process run 3 of max 3, success runs 3/3')
      expect(stdout).toContain('OK  | 1 passed')
      expect(err).toBeNull()
      done()
    })
  })

  it('should display 2 success count of attemps', done => {
    exec(`${codecept_run_config('codecept.conf.min_less_max.js')} --debug`, (err, stdout) => {
      const runs = stdout.split('Run Rerun - Command --')

      // check first run
      expect(runs[2]).toContain('OK  | 1 passed')
      // expect(runs[2]).toContain('✔ OK')

      // check second run
      expect(runs[2]).toContain('OK  | 1 passed')
      // expect(runs[2]).toContain('✔ OK')

      expect(stdout).toContain('Process run 1 of max 3, success runs 1/2')
      expect(stdout).toContain('Process run 2 of max 3, success runs 2/2')
      expect(stdout).not.toContain('Process run 3 of max 3')
      expect(stdout).toContain('OK  | 1 passed')
      expect(err).toBeNull()
      done()
    })
  })

  it('should display error if minSuccess more than maxReruns', done => {
    exec(`${codecept_run_config('codecept.conf.min_more_max.js')} --debug`, (err, stdout) => {
      expect(stdout).toContain('minSuccess must be less than maxReruns')
      expect(err.code).toBe(1)
      done()
    })
  })

  it('should display errors if test is fail always', done => {
    exec(`${codecept_run_config('codecept.conf.fail_test.js', '@RunRerun - Fail all attempt')} --debug`, (err, stdout) => {
      expect(stdout).toContain('Fail run 1 of max 3, success runs 0/2')
      expect(stdout).toContain('Fail run 2 of max 3, success runs 0/2')
      expect(stdout).toContain('Fail run 3 of max 3, success runs 0/2')
      expect(stdout).toContain('Flaky tests detected!')
      expect(err.code).toBe(1)
      done()
    })
  })

  it('should display success run if test was fail one time of two attempts and 3 reruns', done => {
    exec(`FAIL_ATTEMPT=0  ${codecept_run_config('codecept.conf.fail_test.js', '@RunRerun - fail second test')} --debug`, (err, stdout) => {
      expect(stdout).toContain('Process run 1 of max 3, success runs 1/2')
      expect(stdout).toContain('Fail run 2 of max 3, success runs 1/2')
      expect(stdout).toContain('Process run 3 of max 3, success runs 2/2')
      expect(stdout).not.toContain('Flaky tests detected!')
      expect(err).toBeNull()
      done()
    })
  })

  it('should throw exit code 1 if all tests were supposed to pass', done => {
    exec(`FAIL_ATTEMPT=0  ${codecept_run_config('codecept.conf.pass_all_test.js', '@RunRerun - fail second test')} --debug`, (err, stdout) => {
      expect(stdout).toContain('Process run 1 of max 3, success runs 1/3')
      expect(stdout).toContain('Fail run 2 of max 3, success runs 1/3')
      expect(stdout).toContain('Process run 3 of max 3, success runs 2/3')
      expect(stdout).toContain('Flaky tests detected!')
      expect(err.code).toBe(1)
      done()
    })
  })
})