diff --git a/src/proxy/http/HttpTransact.cc b/src/proxy/http/HttpTransact.cc index 8137257f268..36e4daeff4f 100644 --- a/src/proxy/http/HttpTransact.cc +++ b/src/proxy/http/HttpTransact.cc @@ -3989,9 +3989,9 @@ HttpTransact::error_log_connection_failure(State *s, ServerState_t conn_state) host_name = s->unmapped_url.host_get(); } swoc::bwprint(error_bw_buffer, - "CONNECT: attempt fail [{}] to {} for host='{}' " + "CONNECT: attempt fail [{}] to {} for host='{}' sm_id={} " "connection_result={::s} error={::s} retry_attempts={} url='{}'", - HttpDebugNames::get_server_state_name(conn_state), s->current.server->dst_addr, host_name, + HttpDebugNames::get_server_state_name(conn_state), s->current.server->dst_addr, host_name, s->state_machine_id(), swoc::bwf::Errno(s->current.server->connect_result), swoc::bwf::Errno(s->cause_of_death_errno), s->current.retry_attempts.get(), swoc::bwf::FirstOf(url_str, "")); Log::error("%s", error_bw_buffer.c_str()); diff --git a/tests/gold_tests/autest-site/ats_replay.test.ext b/tests/gold_tests/autest-site/ats_replay.test.ext index b731f8a5ea5..913257e7c34 100644 --- a/tests/gold_tests/autest-site/ats_replay.test.ext +++ b/tests/gold_tests/autest-site/ats_replay.test.ext @@ -119,6 +119,21 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti gold_file = diags_log['gold_file'] ts.Disk.diags_log.Content += gold_file + # error_log validation. + error_log = log_validation.get('error_log', {}) + for contains_entry in error_log.get('contains', []): + expression = contains_entry['expression'] + description = contains_entry.get('description', f'Verify error_log contains: {expression}') + ts.Disk.error_log.Content += Testers.ContainsExpression(expression, description) + for excludes_entry in error_log.get('excludes', []): + expression = excludes_entry['expression'] + description = excludes_entry.get('description', f'Verify error_log excludes: {expression}') + ts.Disk.error_log.Content += Testers.ExcludesExpression(expression, description) + # Gold file validation for error_log. + if 'gold_file' in error_log: + gold_file = error_log['gold_file'] + ts.Disk.error_log.Content += gold_file + # access_log validation. access_log = log_validation.get('access_log', {}) if 'gold_file' in access_log: @@ -182,6 +197,8 @@ def ATSReplayTest(obj, replay_file: str): dns = tr.MakeDNServer(name, **process_config) else: dns = tr.MakeDNServer(name, default='127.0.0.1') + if 'records' in dns_config: + dns.addRecords(dns_config['records']) # Proxy Verifier Server configuration. if not 'server' in autest_config: @@ -215,6 +232,7 @@ def ATSReplayTest(obj, replay_file: str): ats_config = autest_config['ats'] enable_tls = ats_config.get('enable_tls', False) metric_checks = ats_config.get('metric_checks', []) + log_validation = ats_config.get('log_validation', None) ats_owner = obj if _requires_persistent_ats(ats_config) else tr ts = configure_ats(ats_owner, server=server, ats_config=ats_config, dns=dns) @@ -272,6 +290,13 @@ def ATSReplayTest(obj, replay_file: str): f'^{re.escape(metric_name)}\\s+{expected_value}$', f'{metric_name} should be {expected_value}') check_tr.StillRunningAfter = ts + # wait for error log + if log_validation and log_validation.get('error_log'): + wait_for_log = obj.AddTestRun('Wait for logs of ' + ats_config.get('name', 'ts')) + wait_for_log.Processes.Default.Command = ( + os.path.join(obj.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' + os.path.join(ts.Variables.LOGDIR, 'error.log')) + wait_for_log.Processes.Default.ReturnCode = 0 + return tr diff --git a/tests/gold_tests/dns/connect_attempts.test.py b/tests/gold_tests/dns/connect_attempts.test.py new file mode 100644 index 00000000000..d46cb32d3d5 --- /dev/null +++ b/tests/gold_tests/dns/connect_attempts.test.py @@ -0,0 +1,31 @@ +''' +Verify Origin Server Connect Attempts Behavior +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify Origin Server Connect Attempts Behavior +''' + +# No retry +Test.ATSReplayTest(replay_file="replay/connect_attempts_rr_no_retry.replay.yaml") + +# max_retries +Test.ATSReplayTest(replay_file="replay/connect_attempts_rr_max_retries.replay.yaml") + +# rr_retries +Test.ATSReplayTest(replay_file="replay/connect_attempts_rr_retries.replay.yaml") diff --git a/tests/gold_tests/dns/gold/connect_attempts_rr_max_retries_error_log.gold b/tests/gold_tests/dns/gold/connect_attempts_rr_max_retries_error_log.gold new file mode 100644 index 00000000000..2172f69d461 --- /dev/null +++ b/tests/gold_tests/dns/gold/connect_attempts_rr_max_retries_error_log.gold @@ -0,0 +1,8 @@ +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 `` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 `` retry_attempts=1 url='http://backend.example.com:``/path/' +`` CONNECT : `` connecting to 0.0.0.1:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=1 `` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=1 `` retry_attempts=1 url='http://backend.example.com:``/path/' +`` CONNECT : `` connecting to 0.0.0.2:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down +`` DNS Error: no valid server http://backend.example.com:``/path/ +`` diff --git a/tests/gold_tests/dns/gold/connect_attempts_rr_no_error_log.gold b/tests/gold_tests/dns/gold/connect_attempts_rr_no_error_log.gold new file mode 100644 index 00000000000..1d67ead48ee --- /dev/null +++ b/tests/gold_tests/dns/gold/connect_attempts_rr_no_error_log.gold @@ -0,0 +1,6 @@ +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 `` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT : `` connecting to 0.0.0.1:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=1 `` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT : `` connecting to 0.0.0.2:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down +`` DNS Error: no valid server http://backend.example.com:``/path/ +`` diff --git a/tests/gold_tests/dns/gold/connect_attempts_rr_retries_error_log.gold b/tests/gold_tests/dns/gold/connect_attempts_rr_retries_error_log.gold new file mode 100644 index 00000000000..c835cd3fa5d --- /dev/null +++ b/tests/gold_tests/dns/gold/connect_attempts_rr_retries_error_log.gold @@ -0,0 +1,8 @@ +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=1 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT : `` connecting to 0.0.0.1:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='2' marking down +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=2 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=3 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/' +`` CONNECT : `` connecting to 0.0.0.2:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='2' marking down +`` DNS Error: no valid server http://backend.example.com:``/path/ +`` diff --git a/tests/gold_tests/dns/replay/connect_attempts_rr_max_retries.replay.yaml b/tests/gold_tests/dns/replay/connect_attempts_rr_max_retries.replay.yaml new file mode 100644 index 00000000000..b1c918ab2ce --- /dev/null +++ b/tests/gold_tests/dns/replay/connect_attempts_rr_max_retries.replay.yaml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Configuration section for autest integration +autest: + description: 'Verify connect attempts behavior - round robin' + + dns: + name: 'dns-rr' + records: {"backend.example.com": ["0.0.0.1", "0.0.0.2"]} + + server: + name: 'server-rr' + + client: + name: 'client-rr' + + ats: + name: 'ts-rr' + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http|hostdb|dns' + proxy.config.http.connect_attempts_rr_retries: 0 + proxy.config.http.connect_attempts_max_retries: 1 + proxy.config.http.connect_attempts_max_retries_down_server: 0 + proxy.config.http.connect_attempts_timeout: 1 + proxy.config.http.down_server.cache_time: 10 + + remap_config: + - from: "http://example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + + log_validation: + error_log: + gold_file: "gold/connect_attempts_rr_max_retries_error_log.gold" + +sessions: +- transactions: + # try 0.0.0.1 + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 1] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # try 0.0.0.2 + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 2] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # The request is expected to hit the down_server cache and immediately receive a 500 response. + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 10] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 500 + + # when down_server.cache_time is expired, try connect attempts + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 20] + delay: 10s + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 diff --git a/tests/gold_tests/dns/replay/connect_attempts_rr_no_retry.replay.yaml b/tests/gold_tests/dns/replay/connect_attempts_rr_no_retry.replay.yaml new file mode 100644 index 00000000000..b0ec1819eb9 --- /dev/null +++ b/tests/gold_tests/dns/replay/connect_attempts_rr_no_retry.replay.yaml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Configuration section for autest integration +autest: + description: 'Verify connect attempts behavior - no retry' + + dns: + name: 'dns-no' + records: {"backend.example.com": ["0.0.0.1", "0.0.0.2"]} + + server: + name: 'server-no' + + client: + name: 'client-no' + + ats: + name: 'ts-no' + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http|hostdb|dns' + proxy.config.http.connect_attempts_rr_retries: 0 + proxy.config.http.connect_attempts_max_retries: 0 + proxy.config.http.connect_attempts_max_retries_down_server: 0 + proxy.config.http.connect_attempts_timeout: 1 + proxy.config.http.down_server.cache_time: 5 + + remap_config: + - from: "http://example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + + log_validation: + error_log: + gold_file: "gold/connect_attempts_rr_no_error_log.gold" + +sessions: +- transactions: + # try 0.0.0.1 + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 1] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # try 0.0.0.2 + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 2] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # This request is expected to hit the down_server cache and immediately receive a 500 response. + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 10] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 500 + + # when down_server.cache_time is expired, try connect attempts + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 20] + delay: 10s + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 diff --git a/tests/gold_tests/dns/replay/connect_attempts_rr_retries.replay.yaml b/tests/gold_tests/dns/replay/connect_attempts_rr_retries.replay.yaml new file mode 100644 index 00000000000..0d040345ffa --- /dev/null +++ b/tests/gold_tests/dns/replay/connect_attempts_rr_retries.replay.yaml @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Configuration section for autest integration +autest: + description: 'Verify connect attempts behavior - round robin' + + dns: + name: 'dns-rrr' + records: {"backend.example.com": ["0.0.0.1", "0.0.0.2"]} + + server: + name: 'server-rrr' + + client: + name: 'client-rrr' + + ats: + name: 'ts-rrr' + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http|hostdb|dns' + proxy.config.http.connect_attempts_rr_retries: 2 + proxy.config.http.connect_attempts_max_retries: 0 + proxy.config.http.connect_attempts_max_retries_down_server: 0 + proxy.config.http.connect_attempts_timeout: 1 + proxy.config.http.down_server.cache_time: 10 + + remap_config: + - from: "http://example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + + log_validation: + error_log: + gold_file: "gold/connect_attempts_rr_retries_error_log.gold" + +sessions: +- transactions: + # try 0.0.0.1 + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 1] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # try 0.0.0.1 again + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 2] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # try 0.0.0.2 + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 3] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # try 0.0.0.2 again + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 4] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502 + + # The request is expected to hit the down_server cache and immediately receive a 500 response. + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 10] + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 500 + + # when down_server.cache_time is expired, try connect attempts + - client-request: + method: GET + url: /path/ + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 20] + delay: 10s + + # should not hit + server-response: + status: 200 + reason: OK + + proxy-response: + status: 502