Update setup_clang_toolchain and requirements (#1476)

One half of the local build/test issue reported by other maintainers.

- fixes _ci.yml to _append_ additional bazel build arguments rather than override the CI defaults. If it's running from _ci.yml, we always want those defaults.
- sets toplevel download option using current .bazelrc names for the release task.
- adds `--config=clang` to build and test Bazel execution options when setup_clang_toolchain is invoked in do_ci.sh. This worked on remote execution because remote execution included e.g. `config=docker-clang` which itself sets `config=clang` separately.
- adds `fix_requirements` to do_ci.sh - the requirements update step also uses Bazel and should be using the same Bazel configuration options we're applying for build and test (specifically, updating requirements is otherwise broken because running the bazel command directly omits `export CARGO_BAZEL_REPIN=true` and breaks on a hash mismatch in the lockfile)
- adds `fix_docs` to the do_ci.sh commands as docs, too, relies on Bazel building to re-generate.
- regenerates docs and requirements using new utilities
- updates the maintainer guide to point towards the do_ci.sh utilities instead of invoking the Bazel commands directly. 
- deletes the obsolesced separate docs utility script.
- updates nighthawk_envoy_updater to use `fix_requirements` and `fix_docs` and adds a missing comma from the copied files list.
- also removes a chunk in .bazelrc that was previously submitted by mistake!

Signed-off-by: Elizabeth Byerly <ebyerly@google.com>

Author: ebyerly

.bazelrc

@@ -2,7 +2,7 @@
 # This is necessary since this needs to be available before we can access                   # unique
 # the Envoy repository contents via Bazel.                                                  # unique
                                                                                             # unique
-build:asan --test_timeout=4800                                                               # unique
+build:asan --test_timeout=4800                                                              # unique
 build:tsan --test_timeout=4800                                                              # unique
 build:test_gcc --test_timeout=4800                                                          # unique
 # See https://github.com/envoyproxy/nighthawk/issues/405                                    # unique
@@ -13,17 +13,7 @@ build:macos --copt -UDEBUG
 test:stress --//test/config:run_stress_tests=True                                           # unique
                                                                                             # unique
 # CI configuration                                                                          # unique
-build:remote-ci-download --config=ci                                                        # unique
-build:remote-ci-download --remote_download_toplevel                                         # unique
-
-# DO NOT SUBMIT -->
-# Explicitly set the --host_action_env for clang build since we are not building            # unique
-# rbe-toolchain-clang that Envoy builds.                                                    # unique
-# This value is the same for different VMs, thus cache hits can be shared among machines.   # unique
-build --host_action_env=PATH=/usr/sbin:/usr/bin:/opt/llvm/bin                               # unique
-# To make our own CI green, we do need that flag on Windows though.
-build:windows --action_env=PATH --host_action_env=PATH
-# <--
+build:download-toplevel --remote_download_outputs=toplevel                                  # unique
 
 #############################################################################
 # startup

.github/workflows/_ci.yml

@@ -13,9 +13,6 @@ on:
         description: "Extra bazel args"
         required: false
         type: string
-        default: >-
-          --config=remote-ci
-          --config=rbe
       docker-in-docker:
         description: "Enable Docker in Docker"
         required: false
@@ -65,6 +62,8 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ github.token }}
         BAZEL_BUILD_EXTRA_OPTIONS: >-
+          --config=remote-ci
+          --config=rbe
           ${{ inputs.bazel-extra }}
         GH_BRANCH: ${{ github.ref }}
         GH_SHA1: ${{ github.sha }}

.github/workflows/nighthawk.yml

@@ -65,5 +65,5 @@ jobs:
     with:
       task: docker
       bazel-extra: >-
-        --config=remote-ci-download
+        --config=download-toplevel
       docker-in-docker: true

ci/do_ci.sh

@@ -90,7 +90,7 @@ function maybe_copy_binaries_to_directory () {
 #######################################
 function do_build () {
     bazel build $BAZEL_BUILD_OPTIONS //:nighthawk
-    tools/update_cli_readme_documentation.sh --mode check
+    do_fix_docs
     maybe_copy_binaries_to_directory
 }
 
@@ -183,6 +183,8 @@ function setup_clang_toolchain() {
     export CXX=clang++
     export ASAN_SYMBOLIZER_PATH=/opt/llvm/bin/llvm-symbolizer
     export BAZEL_COMPILER=clang
+    BAZEL_BUILD_OPTIONS="$BAZEL_BUILD_OPTIONS --config=clang"
+    BAZEL_TEST_OPTIONS="$BAZEL_TEST_OPTIONS --config=clang"
     echo "$CC/$CXX toolchain configured"
 }
 
@@ -286,6 +288,23 @@ function do_fix_format() {
     ./tools/format_python_tools.sh fix
 }
 
+function do_fix_requirements() {
+    echo "fix_requirements..."
+    cd "${SRCDIR}"
+    # Wipe the requirements file to prevent caching.
+    echo "" > tools/base/requirements.txt
+    bazel run //tools/base:requirements.update
+}
+
+function do_fix_docs() {
+    echo "fix_docs..."
+    cd "${SRCDIR}"
+    bazel run $BAZEL_BUILD_OPTIONS //tools:update_cli_readme_documentation -- --binary bazel-bin/nighthawk_client --readme README.md --mode=fix
+    bazel run $BAZEL_BUILD_OPTIONS //tools:update_cli_readme_documentation -- --binary bazel-bin/nighthawk_service --readme README.md --mode=fix
+    bazel run $BAZEL_BUILD_OPTIONS //tools:update_cli_readme_documentation -- --binary bazel-bin/nighthawk_output_transform --readme README.md --mode=fix
+    bazel run $BAZEL_BUILD_OPTIONS //tools:update_cli_readme_documentation -- --binary bazel-bin/nighthawk_test_server --readme source/server/README.md --mode=fix
+}
+
 if grep 'docker\|lxc' /proc/1/cgroup; then
     export BUILD_DIR=/build
     echo "Running in Docker, built binaries will be copied into ${BUILD_DIR}."
@@ -368,6 +387,16 @@ case "$1" in
         do_fix_format
         exit 0
     ;;
+    fix_requirements)
+        setup_clang_toolchain
+        do_fix_requirements
+        exit 0
+    ;;
+    fix_docs)
+        setup_clang_toolchain
+        do_fix_docs
+        exit 0
+    ;;
     benchmark_with_own_binaries)
         setup_clang_toolchain
         do_benchmark_with_own_binaries
@@ -384,7 +413,7 @@ case "$1" in
         exit 0
     ;;
     *)
-        echo "must be one of [opt_build,build,test,clang_tidy,coverage,coverage_integration,asan,tsan,benchmark_with_own_binaries,docker,check_format,fix_format,test_gcc]"
+        echo "must be one of [opt_build,build,test,clang_tidy,coverage,coverage_integration,asan,tsan,benchmark_with_own_binaries,docker,check_format,fix_format,fix_requirements,fix_docs,test_gcc]"
         exit 1
     ;;
 esac

docs/root/updating_envoy_dependency.md

@@ -280,7 +280,7 @@ These should always be accompanied with a comment explaining them. Avoid using
 Update the dependencies by running:
 
 ```bash
-bazel run //tools/base:requirements.update
+ci/do_ci.sh fix_requirements
 ```
 
 This will use the configuration from
@@ -342,7 +342,7 @@ and retrying the format command.
 If Nighthawk command line flags have been changed, execute:
 
 ```bash
-tools/update_cli_readme_documentation.sh --mode fix
+ci/do_ci.sh fix_docs
 ```
 
 to regenerate the