robot.yaml

name: WeWork Robot

# Controls when the workflow will run
on:
  workflow_call:
    secrets:
      TOKEN_ACTION:
        description: "TOKEN_ACTION"
        required: true
      WECHAT_WORK_BOT_WEBHOOK:
        description: "WECHAT_WORK_BOT_WEBHOOK"
        required: true
      DOCU_GROUP_HOOK:
        description: "DOCU_GROUP_HOOK"
        required: true

# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
jobs:
  issue-notify:
    if: github.event_name == 'issues' && github.event.action == 'assigned'
    environment: ci
    runs-on: ubuntu-22.04
    steps:
      - name: ISSUE Triggered
        uses: chf007/action-wechat-work@master
        env:
          WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}}
        with:
          msgtype: markdown
          mentioned_list: '["@all"]'
          content: "New Issue Opened or Assigned: [${{ github.event.issue.title }}](${{ github.event.issue.html_url }})\nAssignees:<font color=\"warning\">${{ join(github.event.issue.assignees.*.login, ', ') }}</font>\nOpen Issue Number: <font color=\"warning\">${{ github.event.repository.open_issues }}</font>"


  check-bvt-issue:
    if: github.event_name == 'issues' && github.event.action == 'closed'
    environment: ci
    runs-on: ubuntu-22.04
    steps:
      - name: CheckOut
        uses: actions/checkout@v6
        with:
          token: ${{ secrets.TOKEN_ACTION }}

      - name: Check the code to see if the issue number is removed
        id: get_check_val
        env:
          ISSUE_NUM: ${{ github.event.issue.number }}
        run: |
          res=`grep -r "issue#$ISSUE_NUM" ${{ github.workspace }}/test || true`
          if [ -n "$res" ]; then
            echo "::set-output name=check_val::True"
          else
            echo "::set-output name=check_val::False"
          fi

      - name: Reopen Issue
        if: ${{ steps.get_check_val.outputs.check_val == 'True' }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: "open-issue"
          token: ${{ secrets.TOKEN_ACTION }}
          issue-number: ${{ github.event.issue.number }}

      - name: create-comment
        if: ${{ steps.get_check_val.outputs.check_val == 'True' }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: "create-comment"
          token: ${{ secrets.TOKEN_ACTION }}
          issue-number: ${{ github.event.issue.number }}
          body: |
            Hello @${{ github.event.issue.user.login }}. The bug issue in the BVT test code has not been removed,issues automatically open.

  issue-reopen:
    environment: ci
    runs-on: ubuntu-22.04
    if: github.event_name == 'issues' && github.event.action == 'closed'
    steps:
      - name: Reopen Issue
        if: github.event.issue.user.login != github.event.sender.login && github.event.action == 'closed' && github.event.sender.login != 'sukki37' && github.event.sender.login != 'Ariznawlll' && github.event.sender.login != 'fengttt' && github.event.sender.login != 'heni02'
        uses: actions-cool/issues-helper@v3
        with:
          actions: "open-issue"
          token: ${{ secrets.TOKEN_ACTION }}
          issue-number: ${{ github.event.issue.number }}

  issue_close_check:
    environment: ci
    runs-on: ubuntu-22.04
    if: github.event_name == 'issues' && github.event.action == 'closed'
    steps:
      - uses: guguducken/pull-issue-reviewer@issue_close_check_with_name
        with:
          token_action: ${{ secrets.TOKEN_ACTION }}
          uri_notice: ${{ secrets.DOCU_GROUP_HOOK }}
          name_label: "doc influence"
          mentions: "Prinz,DengNan"


  nightly-check:
    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    environment: ci
    runs-on: amd64-mo-shanghai-4xlarge32
    steps:
      - uses: actions/checkout@v6
        with:
          token: ${{ secrets.TOKEN_ACTION }}
          fetch-depth: "3"
          path: ./matrixone
      - name: Set up Go
        uses: matrixorigin/CI/actions/setup-env@main
        with:
          setup-java: false
          go-version-file: "${{ github.workspace }}/matrixone/go.mod"
      - name: Set Variables
        run: |
          echo "ut_report='UT-Report.out'" >> $GITHUB_ENV
          echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV
          echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV
          echo "ut_coverage='ut_coverage.out'" >> $GITHUB_ENV
          echo "bvt_coverage='bvt_coverage.out'" >> $GITHUB_ENV
          echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV
          echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV
          echo "ut_excluded_pkg='pkg/pb\|pkg/sql/parsers/goyacc\|yaccpar'" >> $GITHUB_ENV
          echo "bvt_excluded_pkg='pkg/pb\|yaccpar'" >> $GITHUB_ENV
          echo 'GOPROXY=http://goproxy.goproxy.svc.cluster.local|https://goproxy.cn|direct' >> $GITHUB_ENV
          echo "$JAVA_HOME/bin" >> $GITHUB_PATH;
      - name: Clone test-tool repository
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-tester
          path: ./mo-tester
          ref: main
      - name: Add cn.txn
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          echo "[cn.txn]" >> ./etc/launch/cn.toml
          echo "  enable-leak-check = 1" >> ./etc/launch/cn.toml
          echo '  max-active-ages = "2m"'>> ./etc/launch/cn.toml
      - name: Start Unit Test
        id: ut_test
        timeout-minutes: 120
        run: |
          set -o pipefail
          cd $GITHUB_WORKSPACE/matrixone
          test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog')
          make clean && make config && make cgo
          CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo"
          CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo"
          THIRDPARTIES_DIR=$GITHUB_WORKSPACE/matrixone/thirdparties
          THIRDPARTIES_INSTALL_DIR=$THIRDPARTIES_DIR/install
          if [ -d $THIRDPARTIES_DIR ] ; then
            make thirdparties
            CGO_CFLAGS="${CGO_CFLAGS} -I${THIRDPARTIES_INSTALL_DIR}/include"
            CGO_LDFLAGS="${CGO_LDFLAGS} -L${THIRDPARTIES_INSTALL_DIR}/lib -Wl,-rpath,${THIRDPARTIES_INSTALL_DIR}/lib"
          fi

          echo "ut start"
          CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test -json -short -v -tags matrixone_test -p 6 -covermode=set -coverprofile=${{ env.raw_ut_coverage }} -coverpkg=./... ${test_scope} | tee ${{ env.ut_report }}
          echo "ut finished"
      - name: Start BVT Test Service
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 10
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          sed -i.bak 's/level = "info"/level = "error"/g'  ./etc/launch/log.toml
          if  [ -d  "./store"  ]; then rm -rf ./store; fi
          export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone
          make GOBUILD_OPT=-cover build
          ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch
      - name: Update BVT SQL Timeout
        run: |
          cd $GITHUB_WORKSPACE/mo-tester
          sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml
      - name: Start BVT Test
        if: ${{ always() && !cancelled() }}
        id: bvt_on_pr_version
        run: |
          export LC_ALL="C.UTF-8"
          locale

          cd $GITHUB_WORKSPACE/mo-tester
          sed -i 's/  port: [0-9]*/  port: 12345/g' mo.yml
          cat mo.yml
          echo "============================"

          ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1
      - name: Generate UT and BVT Coverage File
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 10
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          cat ${{ env.raw_ut_coverage }} | grep -v ${{ env.ut_excluded_pkg }} > ${{ env.ut_coverage }}
          echo "ut_coverage finished"
          go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.ut_coverage }}
          echo "ut_html_coverage finished"
          rm -rf ${{ env.raw_ut_coverage }}
          ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT
          i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done
          if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi
          go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }}
          cat ${{ env.raw_bvt_coverage }} | grep -v ${{ env.bvt_excluded_pkg }} > ${{ env.bvt_coverage }}
          echo "bvt_coverage finished"
          go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.bvt_coverage }}
          echo "bvt_html_coverage finished"
      - name: Coverage Calculate
        if: ${{ always() && !cancelled() }}
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          echo "UT_CODE_COVERAGE: $(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.ut_coverage }})" > coverage_report.txt
          echo "BVT_CODE_COVERAGE: $(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.bvt_coverage }})"  >> coverage_report.txt
          echo "UT_CODE_COVERAGE=$(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.ut_coverage }})" >> $GITHUB_ENV
          echo "BVT_CODE_COVERAGE=$(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.bvt_coverage }})" >> $GITHUB_ENV
          coverage="$(cat ${{ env.ut_coverage }} ${{ env.bvt_coverage }} | awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}')"
          echo "Overall Coverage: ${coverage}"  >> coverage_report.txt
          echo "GENERAL_CODE_COVERAGE=${coverage}" >> $GITHUB_ENV
          pkgs=$(go list ./pkg/... | awk -v FS='/' -v OFS='/' '{if ($5=="vm" || $5=="sql") print $4,$5,$6"/"; else print $4,$5"/"}' | sort -u | grep -v ${{ env.ut_excluded_pkg }} | grep -v '//')
          echo -e "\n------------ut detail coverage----------"  >> coverage_report.txt
          echo "" > ut_tmp.txt
          for pkg in ${pkgs[@]}
          do
          if [ "$(cat ${{ env.ut_coverage }} |grep $pkg | grep -v "grep" | wc -l)" -gt 0 ];
          then
              c="$(cat ${{ env.ut_coverage }} |grep $pkg|awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}')";
              echo -e "${pkg}:\t${c}" >> ut_tmp.txt;
          else
              echo -e "${pkg}:\t0" >> ut_tmp.txt;
          fi
          done
          cat ut_tmp.txt | sort -k 1 -k 2n  >> coverage_report.txt
          pkgs=$(go list ./pkg/... | awk -v FS='/' -v OFS='/' '{if ($5=="vm" || $5=="sql") print $4,$5,$6"/"; else print $4,$5"/"}' | sort -u | grep -v ${{ env.bvt_excluded_pkg }} | grep -v '//')
          echo -e "\n------------bvt detail coverage----------" >> coverage_report.txt
          echo "" > bvt_tmp.txt
          for pkg in ${pkgs[@]}
          do
          if [ "$(cat ${{ env.bvt_coverage }} |grep $pkg | grep -v "grep" | wc -l)" -gt 0 ];
          then
              c="$(cat ${{ env.bvt_coverage }} |grep $pkg|awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}')";
              echo -e "${pkg}:\t${c}" >> bvt_tmp.txt;
          else
              echo -e "${pkg}:\t0" >> bvt_tmp.txt;
          fi
          done
          cat bvt_tmp.txt | sort -k 1 -k 2n >> coverage_report.txt
          # zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} coverage_report.txt lint.out UT-Report.out
          zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} coverage_report.txt UT-Report.out $GITHUB_WORKSPACE/matrixone/mo-service.log
      - name: Generate Notice Content
        if: ${{ always() && !cancelled() }}
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          cat coverage_report.txt
          echo "Today=$(date -d now +%Y-%m-%d)" >> $GITHUB_ENV
          echo "Url=${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV
          echo 'Detail<<EOF' >> $GITHUB_ENV
          cat coverage_report.txt >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@v7
        if: ${{ always() && !cancelled() }}
        continue-on-error: true
        with:
          name: daily-code-quality-reports
          path: |
            ${{ github.workspace }}/matrixone/code_quality_reports.zip
          retention-days: 7
      - uses: actions/upload-artifact@v7
        if: ${{ failure() }}
        continue-on-error: true
        with:
          name: daily-test-reports
          path: |
            ${{ github.workspace }}/matrixone
          retention-days: 7
      - name: WeChat Work notification
        if: ${{ always() && !cancelled() }}
        id: notification
        uses: guguducken/action-wechat-work@master
        env:
          WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}}
        with:
          msgtype: markdown
          mentioned_list: '["@all"]'
          content: "<font color=\"warning\">Nightly Code Quality - ${{ env.Today }} </font>\nUT result:${{ steps.ut_test.conclusion }}\nBVT result:${{ steps.bvt_on_pr_version.conclusion }}\nUT Coverage Summary:<font color=\"warning\">${{ env.UT_CODE_COVERAGE }}</font>\nBVT Coverage Summary:<font color=\"warning\">${{ env.BVT_CODE_COVERAGE }}</font>\nGeneral Coverage Summary:<font color=\"warning\">${{ env.GENERAL_CODE_COVERAGE }}</font>\nFor details, please visit [Action Run](${{ env.Url }})\n"
      - name: WeChat Work notification
        if: ${{ always() && !cancelled() }}
        id: detail-notification
        uses: guguducken/action-wechat-work@master
        env:
          WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}}
        with:
          msgtype: text
          mentioned_list: '["@all"]'
          content: ${{ env.Detail }}

  issue-rm-labels:
    environment: ci
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'issues' && github.event.action == 'assigned' }}
    steps:
      - name: Remove Labels
        uses: matrixorigin/CI/actions/rm-labels@main
        id: step1
        env:
          BASE_URL: ${{ github.api_url }}
        with:
          repository: ${{ github.repository }} # 获取执行该作业的仓库名 形如: matrixorigin/CI
          issue_number: ${{ github.event.issue.number }} # 获取当前操作的issue number
          github_token: ${{ secrets.TOKEN_ACTION }} # 获取github action token操作权限
          labels: "needs-triage" #要删除的label标签 形如 label1,label2,...
          current_assignee: ${{ github.event.assignee.login }}
          blacklist: "matrix-meow" #黑名单形如: user1,user2,...

  reopen-without-PR:
    environment: ci
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'issues' && github.event.action == 'closed' && github.actor != 'sukki37' && github.actor != 'Ariznawlll' }}
    steps:
      - name: Reopen Issue Without PR
        id: step1
        with:
          whitelist: "sukki37,fengttt,heni02" # 白名单成员 可以无条件删除带有特殊标签的issue
          issue_owner: ${{ github.event.issue.user.login }} # issue的打开者 当满足存在关联pr时允许关闭
          close_user: ${{ github.event.sender.login }} # 当前关闭issue的操作者
          github_token: ${{ secrets.TOKEN_ACTION }} # 获取github action token操作权限
          base_url: ${{ github.api_url }}
          repository: ${{ github.repository }} # 获取执行该作业的仓库名 形如: matrixorigin/CI
          issue_number: ${{ github.event.issue.number }} # 获取当前操作的issue number
          assignees: "sukki37" #issue指定分配对象
          LABELS: "no-pr-linked" #相关自定义label配置 形如 label1,label2
          LABELS_NEED: "kind/bug,Bug fix,bug_fix,bug/ut" # 指定需要排查的标签 label1,label2,...
        uses: matrixorigin/CI/actions/reopen-without-PR@main
      - name: WeChat Work notification
        if: ${{ failure() || cancelled() || steps.step1.outputs.send == 'yes' }}
        id: notification
        uses: chf007/action-wechat-work@master
        env:
          WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}}
        with:
          msgtype: markdown
          content: "MO Close Issue Without PR: [${{ github.event.issue.number }}](${{ github.event.issue.html_url }})\nCreator:<@${{ github.event.issue.user.login }}>"