From 772664bcd95f51d4e467e86898b8289fecd19e9c Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 27 Mar 2026 13:28:26 +0530 Subject: [PATCH] octeon: add octeon virtio support This patch contains following changes: 1.Adds new bus support as virtio. 2.Adds new octeon-virtio driver support. Type: feature Signed-off-by: Bheemappa Agasimundin Signed-off-by: Monendra Singh Kushwaha Change-Id: Ifaf6b0eb4c8822759f311d8bc35e2160e55df3ba --- build/external/Makefile | 3 +- build/external/packages/dpdk.mk | 30 +- build/external/packages/octeon-dao.mk | 62 +++ src/plugins/dev_octeon/CMakeLists.txt | 8 + src/plugins/dev_octeon/dev_octeon_virtio.mk | 56 +++ src/plugins/dev_octeon/dpu/dpu.c | 302 ++++++++++++++ src/plugins/dev_octeon/dpu/dpu.h | 52 +++ src/plugins/dev_octeon/oct_virtio.h | 163 ++++++++ src/plugins/dev_octeon/virtio.c | 434 ++++++++++++++++++++ src/plugins/dev_octeon/virtio_bus.c | 117 ++++++ src/plugins/dev_octeon/virtio_bus.h | 25 ++ src/plugins/dev_octeon/virtio_ctrl.c | 370 +++++++++++++++++ src/plugins/dev_octeon/virtio_format.c | 41 ++ src/plugins/dev_octeon/virtio_port.c | 171 ++++++++ src/plugins/dev_octeon/virtio_rx_node.c | 359 ++++++++++++++++ src/plugins/dev_octeon/virtio_tx_node.c | 228 ++++++++++ 16 files changed, 2392 insertions(+), 29 deletions(-) create mode 100644 build/external/packages/octeon-dao.mk create mode 100644 src/plugins/dev_octeon/dev_octeon_virtio.mk create mode 100644 src/plugins/dev_octeon/dpu/dpu.c create mode 100644 src/plugins/dev_octeon/dpu/dpu.h create mode 100644 src/plugins/dev_octeon/oct_virtio.h create mode 100644 src/plugins/dev_octeon/virtio.c create mode 100644 src/plugins/dev_octeon/virtio_bus.c create mode 100644 src/plugins/dev_octeon/virtio_bus.h create mode 100644 src/plugins/dev_octeon/virtio_ctrl.c create mode 100644 src/plugins/dev_octeon/virtio_format.c create mode 100644 src/plugins/dev_octeon/virtio_port.c create mode 100644 src/plugins/dev_octeon/virtio_rx_node.c create mode 100644 src/plugins/dev_octeon/virtio_tx_node.c diff --git a/build/external/Makefile b/build/external/Makefile index 5394eccc3..82f575390 100644 --- a/build/external/Makefile +++ b/build/external/Makefile @@ -22,6 +22,7 @@ ifneq ($(shell uname), FreeBSD) include packages/rdma-core.mk include packages/xdp-tools.mk include packages/octeon-roc.mk +include packages/octeon-dao.mk endif # ! FreeBSD include packages/dpdk.mk @@ -33,7 +34,7 @@ clean: ifeq ($(shell uname), FreeBSD) install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install quicly-install else -install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install) +install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install octeon-dao-install) endif # FreeBSD .PHONY: config diff --git a/build/external/packages/dpdk.mk b/build/external/packages/dpdk.mk index da6802e48..25945a13c 100644 --- a/build/external/packages/dpdk.mk +++ b/build/external/packages/dpdk.mk @@ -21,11 +21,11 @@ DPDK_MLX_IBV_LINK ?= static # On most of the systems, default value for max lcores is 128 DPDK_MAX_LCORES ?= -dpdk_version ?= 24.11.1 +dpdk_version ?= 25.11 dpdk_base_url ?= http://fast.dpdk.org/rel dpdk_tarball := dpdk-$(dpdk_version).tar.xz -dpdk_tarball_sha256sum_24.11.1 := bcae7d42c449fc456dfb279feabcbe0599a29bebb2fe2905761e187339d96b8e +dpdk_tarball_sha256sum_25.11 := 52e90d2a531ef3ded0283bd91abc94980698f1f6471fa09658a0217cf6609526 dpdk_tarball_sha256sum := $(dpdk_tarball_sha256sum_$(dpdk_version)) dpdk_url := $(dpdk_base_url)/$(dpdk_tarball) @@ -57,12 +57,9 @@ endif DPDK_DRIVERS_DISABLED := baseband/\*, \ bus/dpaa, \ bus/ifpga, \ - common/cnxk, \ compress/isal, \ - compress/octeontx, \ compress/zlib, \ crypto/ccp, \ - crypto/cnxk, \ crypto/dpaa_sec, \ crypto/openssl, \ crypto/aesni_mb, \ @@ -70,16 +67,10 @@ DPDK_DRIVERS_DISABLED := baseband/\*, \ crypto/kasumi, \ crypto/snow3g, \ crypto/zuc, \ - event/\*, \ mempool/dpaa, \ - mempool/cnxk, \ - net/af_packet, \ net/bnx2x, \ - net/bonding, \ - net/cnxk, \ net/ipn3ke, \ net/liquidio, \ - net/pcap, \ net/pfe, \ net/sfc, \ net/softnic, \ @@ -90,27 +81,10 @@ DPDK_DRIVERS_DISABLED := baseband/\*, \ DPDK_LIBS_DISABLED := acl, \ bbdev, \ bitratestats, \ - bpf, \ - cfgfile, \ - cnxk, \ - distributor, \ - efd, \ - fib, \ - flow_classify, \ - graph, \ - gro, \ - gso, \ jobstats, \ kni, \ - latencystats, \ - lpm, \ member, \ - node, \ pipeline, \ - port, \ - power, \ - rawdev, \ - rib, \ table DPDK_MLX_CONFIG_FLAG := diff --git a/build/external/packages/octeon-dao.mk b/build/external/packages/octeon-dao.mk new file mode 100644 index 000000000..fd0ba2aab --- /dev/null +++ b/build/external/packages/octeon-dao.mk @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Marvell. + +DAO_DEBUG ?= n +octeon-dao_version := dao-cn10k-26.02.0-ubuntu-24.04-26.02.0 +octeon-dao_tarball := $(octeon-dao_version).tar.gz +octeon-dao_tarball_sha256sum := 0bd606591bd9d42d6b04d656da2ca4bbf4b5c66762e4acaf93664ad1a505f387 + +octeon-dao_tarball_strip_dirs := 1 +octeon-dao_url := https://github.com/MarvellEmbeddedProcessors/dao/archive/refs/tags/$(octeon-dao_tarball) + +octeon_dao_cmake_args ?= + +DAO_BUILD_TYPE:=release +ifeq ($(DAO_DEBUG), y) +DAO_BUILD_TYPE:=debug +endif + +DAO_MESON_ARGS = \ + --default-library static \ + -Dprefer_static_build=true --prefer-static \ + -Denable_apps=virtio-l2fwd \ + -Denable_libs=virtio,virtio_net,vfio,pem,pal,common \ + --buildtype=$(DAO_BUILD_TYPE) \ + -Denable_kmods=false + +PREFIX = $(OCTEON_SDK_SYSROOT) + +ifeq (,$(findstring $(OCTEON_VERSION),cn10k cn9k)) + DAO_MESON_ARGS += -Dplatform=native + DAO_MESON_ARGS += --prefix $(octeon-dao_install_dir) + PREFIX = $(octeon-dao_install_dir) +else ifeq ($(OCTEON_VERSION), cn10k) + DAO_MESON_ARGS += --cross-file=$(octeon-dao_src_dir)/config/arm64_cn10k_linux_gcc + DAO_MESON_ARGS += --prefix $(OCTEON_SDK_SYSROOT) +else ifeq ($(OCTEON_VERSION), cn9k) + DAO_MESON_ARGS += --cross-file=$(octeon-dao_src_dir)/config/arm64_cn9k_linux_gcc + DAO_MESON_ARGS += --prefix $(OCTEON_SDK_SYSROOT) +endif + +PIP_DOWNLOAD_DIR = $(CURDIR)/downloads/ + +define octeon-dao_config_cmds + PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig meson setup $(octeon-dao_src_dir) \ + $(octeon-dao_build_dir) \ + $(DAO_MESON_ARGS) -Dc_args="-Wno-error" -Dc_link_args="-Wno-error"\ + | tee $(dao_config_log) && \ + echo "DAO post meson configuration" +endef + +define octeon-dao_build_cmds + cd ${octeon-dao_build_dir} && rm -f $(octeon-dao_build_log) && \ + ninja install -C ${octeon-dao_build_dir} | tee $(octeon-dao_build_log) +endef + +define octeon-dao_install_cmds + cd ${octeon-dao_build_dir} && \ + meson install &&\ + echo "meson installed directory ${octeon-dao_install_dir}" +endef + +$(eval $(call package,octeon-dao)) diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 8325b726d..e14fd84a9 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -19,7 +19,13 @@ if (NOT OCTEON_ROC_LIB) return () endif() +if (NOT OCTEON_ROC_DIR OR NOT OCTEON_ROC_LIB) + include (dev_octeon_virtio.mk) + return () +endif() + include_directories (${OCTEON_ROC_DIR}/) +include_directories (${CMAKE_CURRENT_SOURCE_DIR}/dpu/) add_vpp_plugin(dev_octeon SOURCES @@ -39,6 +45,7 @@ add_vpp_plugin(dev_octeon tm.c pfc.c dma.c + dpu/dpu.c MULTIARCH_SOURCES rx_node.c @@ -48,3 +55,4 @@ add_vpp_plugin(dev_octeon ${OCTEON_ROC_LIB} ) +include (dev_octeon_virtio.mk) diff --git a/src/plugins/dev_octeon/dev_octeon_virtio.mk b/src/plugins/dev_octeon/dev_octeon_virtio.mk new file mode 100644 index 000000000..fff3918d5 --- /dev/null +++ b/src/plugins/dev_octeon/dev_octeon_virtio.mk @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright(c) 2024 Marvell. + +# Find OCTEON roc files +vpp_plugin_find_library(dev-octeon-virtio DAO_PAL_LIB "libdao_pal.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_VIRT_LIB "libdao_virtio.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_VIRT_NET_LIB "libdao_virtio_net.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_VFIO_LIB "libdao_vfio.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_PEM_LIB "libdao_pem.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_COMM_LIB "libdao_common.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_DPDK_LIB "libdpdk.a") + +vpp_find_path(DAO_NETDEV_INCLUDE_DIR NAMES dao_virtio_netdev.h) + +if (NOT DAO_NETDEV_INCLUDE_DIR) + message("OCTEON VIRTIO DAO files not found - Marvell OCTEON virtio device plugin disabled") + return() +endif() + +set(DAO_CONFG_INCLUDE_DIR "${DAO_NETDEV_INCLUDE_DIR}/..") + +if (NOT DAO_PAL_LIB OR NOT DAO_VIRT_LIB OR NOT DAO_VIRT_NET_LIB OR NOT DAO_VFIO_LIB OR NOT DAO_PEM_LIB OR NOT DAO_COMM_LIB) + message("OCTEON VIRTIO DAO LIBS are not found - Marvell OCTEON virtio device plugin disabled") + return() +endif() + +unset(DAO_LINK_FLAGS) + +get_filename_component(DAO_DPDK_LIB_DIR ${DAO_DPDK_LIB} DIRECTORY) + +link_directories(${DAO_DPDK_LIB_DIR}) +string_append(DAO_LINK_FLAGS "-L${DAO_DPDK_LIB_DIR}") +string_append(DAO_LINK_FLAGS "-lnuma -lz -lelf -lpcap -ljansson -lfdt") +if(OPENSSL_FOUND) + string_append(DAO_LINK_FLAGS "-lssl") + string_append(DAO_LINK_FLAGS "-lcrypto") +endif() + +string_append(DAO_LINK_FLAGS "-Wl,--whole-archive,${DAO_PAL_LIB},${DAO_VIRT_LIB},${DAO_VIRT_NET_LIB},${DAO_VFIO_LIB},${DAO_PEM_LIB},${DAO_COMM_LIB},${DAO_DPDK_LIB},--no-whole-archive") + +include_directories (${DAO_NETDEV_INCLUDE_DIR}/) +include_directories (${DAO_CONFG_INCLUDE_DIR}/) + +add_vpp_plugin(dev_octeon_virtio + SOURCES + virtio.c + virtio_bus.c + virtio_port.c + virtio_ctrl.c + virtio_tx_node.c + virtio_rx_node.c + virtio_format.c + + LINK_FLAGS + "${DAO_LINK_FLAGS}" +) diff --git a/src/plugins/dev_octeon/dpu/dpu.c b/src/plugins/dev_octeon/dpu/dpu.c new file mode 100644 index 000000000..b04ede403 --- /dev/null +++ b/src/plugins/dev_octeon/dpu/dpu.c @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +/** + * @file + * @brief Host DPU interface. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +always_inline void +h2d_compute_checksum (vlib_main_t *vm, vlib_buffer_t *b) +{ + ethernet_header_t *e; + ip4_header_t *ip; + tcp_header_t *th; + udp_header_t *uh; + + e = vlib_buffer_get_current (b); + if (PREDICT_TRUE (clib_net_to_host_u16 (e->type) == ETHERNET_TYPE_IP4)) + { + ip = (ip4_header_t *) (((u8 *) e) + sizeof (ethernet_header_t)); + if (ip->protocol == IP_PROTOCOL_TCP) + { + th = (tcp_header_t *) (b->data + b->current_data + + sizeof (ethernet_header_t) + + ip4_header_bytes (ip)); + th->checksum = 0; + th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip); + } + else if (ip->protocol == IP_PROTOCOL_UDP) + { + uh = (udp_header_t *) (b->data + b->current_data + + sizeof (ethernet_header_t) + + ip4_header_bytes (ip)); + uh->checksum = 0; + uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip); + } + } +} + +static u8 * +format_h2d_input_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + + s = format (s, "h2d-input:\n"); + return s; +} + +VLIB_NODE_FN (h2d_input_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_left, next0, next1, next2, next3; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; + u16 nexts[VLIB_FRAME_SIZE], *next; + vlib_buffer_t **b = bufs; + u32 *from; + + from = vlib_frame_vector_args (frame); + n_left = frame->n_vectors; + next = nexts; + vlib_get_buffers (vm, from, bufs, n_left); + + while (n_left >= 8) + { + vlib_buffer_advance (b[0], OCT_H2D_META_SIZE); + vlib_buffer_advance (b[1], OCT_H2D_META_SIZE); + vlib_buffer_advance (b[2], OCT_H2D_META_SIZE); + vlib_buffer_advance (b[3], OCT_H2D_META_SIZE); + + h2d_compute_checksum (vm, b[0]); + h2d_compute_checksum (vm, b[1]); + h2d_compute_checksum (vm, b[2]); + h2d_compute_checksum (vm, b[3]); + + vnet_feature_next (&next0, b[0]); + vnet_feature_next (&next1, b[1]); + vnet_feature_next (&next2, b[2]); + vnet_feature_next (&next3, b[3]); + + next[0] = (u16) next0; + next[1] = (u16) next1; + next[2] = (u16) next2; + next[3] = (u16) next3; + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + if (b[1]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[1], sizeof (u32)); + if (b[2]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[2], sizeof (u32)); + if (b[3]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[3], sizeof (u32)); + + b += 4; + next += 4; + n_left -= 4; + } + + while (n_left) + { + vlib_buffer_advance (b[0], OCT_H2D_META_SIZE); + h2d_compute_checksum (vm, b[0]); + vnet_feature_next (&next0, b[0]); + next[0] = (u16) next0; + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + b += 1; + next += 1; + n_left -= 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + return frame->n_vectors; +} + +VNET_FEATURE_INIT (h2d_input_node, static) = { + .arc_name = "port-rx-eth", + .node_name = "h2d-input", + .runs_before = VNET_FEATURES ("ethernet-input"), +}; + +VLIB_REGISTER_NODE (h2d_input_node) = { + .vector_size = sizeof (u32), + .format_trace = format_h2d_input_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, + .n_next_nodes = 0, + .name = "h2d-input", +}; + +always_inline u8 +d2h_validate_checksum (vlib_main_t *vm, vlib_buffer_t *b) +{ + u8 csum = OCT_D2H_CSUM_VERIFIED; + ethernet_header_t *e; + ip4_header_t *ip; + + e = vlib_buffer_get_current (b); + if (PREDICT_TRUE (clib_net_to_host_u16 (e->type) == ETHERNET_TYPE_IP4)) + { + vlib_buffer_advance (b, sizeof (ethernet_header_t)); + ip = vlib_buffer_get_current (b); + + if (ip->protocol == IP_PROTOCOL_TCP || ip->protocol == IP_PROTOCOL_UDP) + { + ip4_tcp_udp_validate_checksum (vm, b); + if (!(b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT)) + csum = OCT_D2H_CSUM_FAILED; + } + vlib_buffer_advance (b, -sizeof (ethernet_header_t)); + } + return csum; +} + +static u8 * +format_d2h_output_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + + s = format (s, "d2h-output\n"); + return s; +} + +VLIB_NODE_FN (d2h_output_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + oct_d2h_meta_t *hdr0, *hdr1, *hdr2, *hdr3; + u32 n_left, next0, next1, next2, next3; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; + u16 nexts[VLIB_FRAME_SIZE], *next; + u8 csum0, csum1, csum2, csum3; + vlib_buffer_t **b = bufs; + u32 *from; + + from = vlib_frame_vector_args (frame); + n_left = frame->n_vectors; + next = nexts; + vlib_get_buffers (vm, from, bufs, n_left); + + while (n_left >= 8) + { + csum0 = d2h_validate_checksum (vm, b[0]); + csum1 = d2h_validate_checksum (vm, b[1]); + csum2 = d2h_validate_checksum (vm, b[2]); + csum3 = d2h_validate_checksum (vm, b[3]); + + vlib_buffer_advance (b[0], -OCT_D2H_META_SIZE); + vlib_buffer_advance (b[1], -OCT_D2H_META_SIZE); + vlib_buffer_advance (b[2], -OCT_D2H_META_SIZE); + vlib_buffer_advance (b[3], -OCT_D2H_META_SIZE); + + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[4]) - + OCT_D2H_META_SIZE); + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[5]) - + OCT_D2H_META_SIZE); + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[6]) - + OCT_D2H_META_SIZE); + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[7]) - + OCT_D2H_META_SIZE); + + hdr0 = vlib_buffer_get_current (b[0]); + hdr1 = vlib_buffer_get_current (b[1]); + hdr2 = vlib_buffer_get_current (b[2]); + hdr3 = vlib_buffer_get_current (b[3]); + + hdr0->as_u64 = 0; + hdr1->as_u64 = 0; + hdr2->as_u64 = 0; + hdr3->as_u64 = 0; + + hdr0->csum_verified = csum0; + hdr1->csum_verified = csum1; + hdr2->csum_verified = csum2; + hdr3->csum_verified = csum3; + + vnet_feature_next (&next0, b[0]); + vnet_feature_next (&next1, b[1]); + vnet_feature_next (&next2, b[2]); + vnet_feature_next (&next3, b[3]); + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + if (b[1]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[1], sizeof (u32)); + if (b[2]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[2], sizeof (u32)); + if (b[3]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[3], sizeof (u32)); + + next[0] = (u16) next0; + next[1] = (u16) next1; + next[2] = (u16) next2; + next[3] = (u16) next3; + + b += 4; + next += 4; + n_left -= 4; + } + while (n_left) + { + csum0 = d2h_validate_checksum (vm, b[0]); + vlib_buffer_advance (b[0], -OCT_D2H_META_SIZE); + hdr0 = vlib_buffer_get_current (b[0]); + hdr0->as_u64 = 0; + hdr0->csum_verified = csum0; + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + vnet_feature_next (&next0, b[0]); + next[0] = (u16) next0; + + b += 1; + next += 1; + n_left -= 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + return frame->n_vectors; +} + +VNET_FEATURE_INIT (d2h_output_node, static) = { + .arc_name = "interface-output", + .node_name = "d2h-output", + .runs_before = VNET_FEATURES ("interface-output-arc-end"), +}; + +VLIB_REGISTER_NODE (d2h_output_node) = { + .vector_size = sizeof (u32), + .format_trace = format_d2h_output_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, + .n_next_nodes = 0, + .name = "d2h-output", +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/dev_octeon/dpu/dpu.h b/src/plugins/dev_octeon/dpu/dpu.h new file mode 100644 index 000000000..15586615b --- /dev/null +++ b/src/plugins/dev_octeon/dpu/dpu.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _DPU_H_ +#define _DPU_H_ + +/** + * HOST to DPU meta data + */ +typedef struct oct_h2d_meta +{ + u64 as_u64[3]; +} oct_h2d_meta_t; + +#define OCT_H2D_META_SIZE (sizeof (oct_h2d_meta_t)) + +/** + * DPU to HOST meta data + */ +typedef union oct_d2h_meta +{ + u64 as_u64; + struct + { + u64 request_id : 16; + u64 reserved : 2; + u64 csum_verified : 2; + u64 destqport : 22; + u64 sport : 6; + u64 opcode : 16; + }; +} oct_d2h_meta_t; + +#define OCT_D2H_META_SIZE (sizeof (oct_d2h_meta_t)) + +#define OCT_D2H_CSUM_FAILED 0x0 +#define OCT_D2H_L4SUM_VERIFIED 0x1 +#define OCT_D2H_IPSUM_VERIFIED 0x2 +#define OCT_D2H_CSUM_VERIFIED (OCT_D2H_L4SUM_VERIFIED | OCT_D2H_IPSUM_VERIFIED) + +#endif /* _DPU_H_ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/dev_octeon/oct_virtio.h b/src/plugins/dev_octeon/oct_virtio.h new file mode 100644 index 000000000..91e03df9e --- /dev/null +++ b/src/plugins/dev_octeon/oct_virtio.h @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ +#ifndef _OCTEON_VIRTIO_H_ +#define _OCTEON_VIRTIO_H_ + +#undef always_inline + +#include +#include +#include +#include +#include +#include + +#define always_inline static inline __attribute__ ((__always_inline__)) + +#include +#include +#include +#include +#include + +#define VIRTIO_NET_RSS_RETA_SIZE 128 +#define OCT_VIRTIO_DEVICE_ID 0xa70d +#define MAX_JUMBO_PKT_LEN 9600 + +#define OCT_ETH_TX_OFFLOAD_IPV4_CKSUM (1 << 0) +#define OCT_ETH_RX_OFFLOAD_CHECKSUM (1 << 1) +#define OCT_ETH_TX_OFFLOAD_TCP_TSO (1 << 2) + +#define foreach_oct_virt_tx_node_counter \ + _ (ENQUE_FAIL, enque_fail, ERROR, "Virtio enqueue failed") + +typedef enum +{ +#define _(f, n, s, d) OCT_VIRT_TX_NODE_CTR_##f, + foreach_oct_virt_tx_node_counter +#undef _ +} oct_tx_node_counter_t; + +typedef struct +{ + u32 sw_if_index; + u16 virtio_id; + u64 tx_q_map; +} oct_virt_tx_trace_t; + +typedef struct +{ + u32 sw_if_index; + u16 virtio_id; + u16 queue_id; + u64 rx_q_map; +} oct_virt_rx_trace_t; + +always_inline vlib_buffer_t * +oct_virt_to_bp (void *b, u16 hdr_len) +{ + return ( + vlib_buffer_t *) ((u8 *) b + + sizeof (((struct dao_virtio_net_hdr *) 0)->desc_data) + + hdr_len - sizeof (vlib_buffer_t)); +} + +always_inline void * +oct_bp_to_virt (vlib_buffer_t *b, u16 hdr_len) +{ + return (void *) ((u8 *) vlib_buffer_get_current (b) - + sizeof (((struct dao_virtio_net_hdr *) 0)->desc_data) - + hdr_len); +} + +typedef struct +{ + u8 status : 1; + u8 full_duplex : 1; + u16 virtio_id; + u32 pem_devid; + u32 speed; +} oct_virtio_device_t; + +typedef struct +{ + u16 reta_size; + u16 vchan_id; + u16 virtio_id; +} oct_virtio_port_t; + +typedef struct +{ + u64 wrkr_cpu_mask; + u64 netdev_map; + u16 netdev_qp_count[DAO_VIRTIO_DEV_MAX]; + u8 dao_lib_initialized; + u8 ip4_csum_offload_enable; +} oct_virtio_main_t; + +typedef struct +{ + u8 state; +} oct_virtio_port_map_t; + +typedef struct +{ + u64 qmap; + u16 last_rx_q; + u16 last_tx_q; + u16 virtio_hdr_sz; +} oct_virtio_q_info_t; + +typedef struct +{ + u64 rx_offloads; + u64 tx_offloads; +} oct_intf_offload_t; + +typedef struct +{ + u8 initialized; + u16 service_core; + u64 netdev_map; + oct_intf_offload_t intf[DAO_VIRTIO_DEV_MAX]; + oct_virtio_q_info_t q_map[DAO_VIRTIO_DEV_MAX]; +} oct_virtio_per_thread_data_t; + +int oct_virtio_dev_status_cb (u16 virtio_devid, u8 status); +int oct_virito_rss_reta_configure (u16 virtio_devid, + struct virtio_net_ctrl_rss *rss); +int oct_virtio_configure_promisc (u16 virtio_devid, u8 enable); +int oct_virtio_configure_allmulti (u16 virtio_devid, u8 enabl); +int oct_virtio_mac_addr_set (u16 virtio_devid, u8 *mac); +int oct_virtio_mac_addr_add (u16 virtio_devid, + struct virtio_net_ctrl_mac *mac_tbl, u8 type); +int oct_virtio_mq_configure (u16 virtio_devid, bool qmap_set); +int oct_virtio_vlib_buffer_alloc (u16 devid, void *buffs[], u16 nb_buffs); +int oct_virtio_vlib_buffer_free (u16 devid, void *buffs[], u16 nb_buffs); + +vnet_dev_rv_t oct_virtio_port_init (vlib_main_t *vm, vnet_dev_port_t *port); +void oct_virtio_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port); + +vnet_dev_rv_t oct_virtio_port_start (vlib_main_t *vm, vnet_dev_port_t *port); + +void oct_virtio_port_stop (vlib_main_t *vm, vnet_dev_port_t *port); + +format_function_t format_oct_virt_rx_trace; +format_function_t format_oct_virt_tx_trace; +u8 *format_oct_virt_port_status (u8 *s, va_list *args); +void oct_virt_buffer_pool_dma_map (vlib_main_t *vm); + +#define log_debug(fmt, ...) \ + vlib_log_debug (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_info(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_notice(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_warn(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_err(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) + +#endif /* _OCTEON_VIRTIO_H_ */ diff --git a/src/plugins/dev_octeon/virtio.c b/src/plugins/dev_octeon/virtio.c new file mode 100644 index 000000000..878031883 --- /dev/null +++ b/src/plugins/dev_octeon/virtio.c @@ -0,0 +1,434 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define OCTEON_VIRTIO_DEV "Marvell Octeon virtio network device" + +oct_virtio_main_t *oct_virtio_main = NULL; +oct_virtio_port_map_t *virtio_port_map = NULL; +oct_virtio_per_thread_data_t *oct_virt_thread_data = NULL; + +VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { + .class_name = "octeon", + .subclass_name = "virtio_init", +}; + +enum oct_virtio_dev_args_types +{ + DEV_ARG_VIRT_NB_VIRTIO_DEVICES = 1, + DEV_ARG_VIRT_DMA_DEVICE_LIST, + DEV_ARG_VIRT_MISC_DEVICE, + DEV_ARG_VIRT_CSUM_OFFLD_EN, + DEV_ARG_VIRT_END +}; + +static vnet_dev_arg_t oct_virtio_dev_args[] = { + { + .id = DEV_ARG_VIRT_NB_VIRTIO_DEVICES, + .name = "nb_virtio", + .desc = "Number of virtio device", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 1, + }, + { + .id = DEV_ARG_VIRT_DMA_DEVICE_LIST, + .name = "dma", + .desc = "DMA device list", + .type = VNET_DEV_ARG_TYPE_STRING, + }, + { + .id = DEV_ARG_VIRT_MISC_DEVICE, + .name = "misc", + .desc = "Miscellaneous device list", + .type = VNET_DEV_ARG_TYPE_STRING, + }, + { + .id = DEV_ARG_VIRT_CSUM_OFFLD_EN, + .name = "enable_csum_offld", + .desc = "Enable Host checksum offload", + .type = VNET_DEV_ARG_TYPE_BOOL, + .default_val.boolean = 0, + }, + { + .id = DEV_ARG_VIRT_END, + .name = "end", + .desc = "Argument end", + .type = VNET_DEV_ARG_END, + }, +}; + +#define _(f, n, s, d) \ + { .name = #n, .desc = d, .severity = VL_COUNTER_SEVERITY_##s }, + +vlib_error_desc_t oct_virtio_tx_node_counters[] = { + foreach_oct_virt_tx_node_counter +}; +#undef _ + +vnet_dev_node_t oct_virtio_rx_node = { + .format_trace = format_oct_virt_rx_trace, +}; + +vnet_dev_node_t oct_virtio_tx_node = { + .format_trace = format_oct_virt_tx_trace, + .error_counters = oct_virtio_tx_node_counters, + .n_error_counters = ARRAY_LEN (oct_virtio_tx_node_counters), +}; + +void +oct_virt_buffer_pool_dma_map (vlib_main_t *vm) +{ + uword i; + size_t page_sz; + vlib_physmem_map_t *pm; + vlib_buffer_pool_t *bp; + int iova_mode = rte_eal_iova_mode (); + + vec_foreach (bp, vm->buffer_main->buffer_pools) + { + if (bp->start) + { + pm = vlib_physmem_get_map (vm, bp->physmem_map_index); + page_sz = 1ULL << pm->log2_page_size; + for (i = 0; i < pm->n_pages; i++) + { + char *va = ((char *) pm->base) + i * page_sz; + uword pa = (iova_mode == RTE_IOVA_VA) ? pointer_to_uword (va) : + pm->page_table[i]; + + dao_pal_vfio_dma_map (pointer_to_uword (va), pa, page_sz); + } + } + } +} + +static clib_error_t * +dao_log_read (clib_file_t *uf) +{ + unformat_input_t input; + u8 *line, *s = 0; + int n, n_try; + + n = n_try = 4096; + while (n == n_try) + { + uword len = vec_len (s); + vec_resize (s, len + n_try); + + n = read (uf->file_descriptor, s + len, n_try); + if (n < 0 && errno != EAGAIN) + return clib_error_return_unix (0, "read"); + vec_set_len (s, len + (n < 0 ? 0 : n)); + } + + unformat_init_vector (&input, s); + + while (unformat_user (&input, unformat_line, &line)) + { + vec_add1 (line, 0); + vec_pop (line); + clib_warning ("%v", line); + vec_free (line); + } + + unformat_free (&input); + return 0; +} + +static void +dao_lib_logging (void) +{ + int log_fds[2] = { 0 }; + + if (pipe (log_fds) == 0) + { + if (fcntl (log_fds[0], F_SETFL, O_NONBLOCK) == 0 && + fcntl (log_fds[1], F_SETFL, O_NONBLOCK) == 0) + { + FILE *f = fdopen (log_fds[1], "a"); + if (f && rte_openlog_stream (f) == 0) + { + clib_file_t t = { 0 }; + t.read_function = dao_log_read; + t.file_descriptor = log_fds[0]; + t.description = format (0, "DAO logging pipe"); + clib_file_add (&file_main, &t); + } + } + else + { + close (log_fds[0]); + close (log_fds[1]); + } + } +} + +static u8 * +oct_virtio_probe (vlib_main_t *vm, vnet_dev_bus_index_t bus_index, + void *dev_info) +{ + oct_dev_bus_virtio_device_info_t *di = dev_info; + + if (di->vendor_id != 0x177d || di->device_id != OCT_VIRTIO_DEVICE_ID) + return 0; + + return format (0, "%s", OCTEON_VIRTIO_DEV); +} + +static char ** +oct_populate_dma_device_list (u16 *nb_elem, u8 *dma_list) +{ + char *device = NULL; + char **vec = NULL; + char *saveptr; + u16 count = 0; + + device = strtok_r ((char *) dma_list, ",", &saveptr); + while (device) + { + vec = reallocarray (vec, count + 1, sizeof (vec)); + vec[count] = strdup (device); + count++; + device = strtok_r (saveptr, ",", &saveptr); + } + *nb_elem = count; + return vec; +} + +static void +oct_virtio_parse_arguments (dao_pal_global_conf_t *conf, vnet_dev_arg_t *args) +{ + vnet_dev_arg_t *a = args; + + for (; a < vec_end (args) && a->val_set; a++) + { + switch (a->id) + { + case DEV_ARG_VIRT_NB_VIRTIO_DEVICES: + conf->nb_virtio_devs = vnet_dev_arg_get_uint32 (a); + break; + case DEV_ARG_VIRT_DMA_DEVICE_LIST: + conf->dma_devices = oct_populate_dma_device_list ( + &conf->nb_dma_devs, vnet_dev_arg_get_string (a)); + break; + case DEV_ARG_VIRT_MISC_DEVICE: + conf->misc_devices = oct_populate_dma_device_list ( + &conf->nb_misc_devices, vnet_dev_arg_get_string (a)); + break; + case DEV_ARG_VIRT_CSUM_OFFLD_EN: + oct_virtio_main->ip4_csum_offload_enable = vnet_dev_arg_get_bool (a); + break; + default: + log_info ("Invalid virtio device arguments received\n"); + } + } +} + +static vnet_dev_rv_t +oct_virtio_init (vlib_main_t *vm, vnet_dev_t *dev) +{ + u8 mac_addr[6]; + vnet_dev_rv_t rv; + uint64_t lcore_mask; + oct_virtio_port_t ovp = {}; + dao_pal_global_conf_t conf = { 0 }; + struct dao_virtio_netdev_cbs cbs = {}; + oct_dev_bus_virtio_device_data_t *bus_data; + oct_virtio_device_t *device_data = vnet_dev_get_data (dev); + + bus_data = oct_get_bus_virtio_device_data (dev); + + if (!oct_virtio_main->dao_lib_initialized) + { + /** + * The initialization of the DAO library will be carried out using the + * arguments provided during the first initialization of the virtio + * interface. Any arguments provided from the second virtio device + * onwards will be disregarded. + */ + oct_virtio_parse_arguments (&conf, dev->args); + + if (dao_pal_global_init (&conf)) + { + log_err ("dao_pal_global_init failed\n"); + return VNET_DEV_ERR_UNSUPPORTED_CONFIG; + } + + /* Update lcore_mask with main core */ + lcore_mask = DAO_BIT_ULL (vm->cpu_id) | oct_virtio_main->wrkr_cpu_mask; + + log_debug ("lcore_mask %lu\n", lcore_mask); + if (dao_pal_dma_dev_setup (lcore_mask)) + { + log_err ("dao_pal_dma_dev_setup failed\n"); + rv = VNET_DEV_ERR_UNSUPPORTED_CONFIG; + goto finish; + } + + /* Set main core DMA devices for virtio control */ + if (dao_pal_dma_ctrl_dev_set (vm->cpu_id)) + { + log_err ("dao_pal_dma_dev_setup failed\n"); + rv = VNET_DEV_ERR_UNSUPPORTED_CONFIG; + goto finish; + } + + oct_virt_buffer_pool_dma_map (vm); + + cbs.status_cb = oct_virtio_dev_status_cb; + cbs.rss_cb = oct_virito_rss_reta_configure; + cbs.promisc_cb = oct_virtio_configure_promisc; + cbs.allmulti_cb = oct_virtio_configure_allmulti; + cbs.mac_set = oct_virtio_mac_addr_set; + cbs.mac_add = oct_virtio_mac_addr_add; + cbs.mq_configure = oct_virtio_mq_configure; + cbs.extbuf_get = oct_virtio_vlib_buffer_alloc; + cbs.extbuf_put = oct_virtio_vlib_buffer_free; + + dao_virtio_netdev_cb_register (&cbs); + + oct_virtio_main->dao_lib_initialized = 1; + } + + ethernet_mac_address_generate (mac_addr); + + device_data->virtio_id = bus_data->virtio_dev.virtio_id; + ovp.virtio_id = bus_data->virtio_dev.virtio_id; + ovp.reta_size = VIRTIO_NET_RSS_RETA_SIZE; + + vnet_dev_port_add_args_t port_add_args = { + .port = { + .attr = { + .type = VNET_DEV_PORT_TYPE_ETHERNET, + .max_rx_queues = DAO_VIRTIO_MAX_QUEUES, + .max_tx_queues = DAO_VIRTIO_MAX_QUEUES, + .max_supported_rx_frame_size = MAX_JUMBO_PKT_LEN, + .caps = { + .rss = 1, + }, + .rx_offloads = { + .ip4_cksum = 1, + }, + }, + .ops = { + .init = oct_virtio_port_init, + .deinit = oct_virtio_port_deinit, + .start = oct_virtio_port_start, + .stop = oct_virtio_port_stop, + .config_change = NULL, + .format_status = format_oct_virt_port_status, + }, + .data_size = sizeof (oct_virtio_port_t), + .initial_data = &ovp, + }, + .rx_node = &oct_virtio_rx_node, + .tx_node = &oct_virtio_tx_node, + .rx_queue = { + .config = { + .data_size = 0, + .default_size = 1024, + .multiplier = 32, + .min_size = 256, + .max_size = 16384, + }, + .ops = { + .alloc = NULL, + .free = NULL, + .format_info = NULL, + }, + }, + .tx_queue = { + .config = { + .data_size = 0, + .default_size = 1024, + .multiplier = 32, + .min_size = 256, + .max_size = 16384, + }, + .ops = { + .alloc = NULL, + .free = NULL, + .format_info = NULL, + }, + }, + }; + + vnet_dev_set_hw_addr_eth_mac (&port_add_args.port.attr.hw_addr, mac_addr); + + log_info ("MAC address is %U", format_ethernet_address, mac_addr); + + rv = vnet_dev_port_add (vm, dev, 0, &port_add_args); + + return rv; + +finish: + dao_pal_global_fini (); + return rv; +} + +static clib_error_t * +oct_virtio_worker_init (vlib_main_t *vm) +{ + u16 cpu_id = clib_get_current_cpu_id (); + + oct_virtio_main->wrkr_cpu_mask |= DAO_BIT_ULL (cpu_id); + + return 0; +} + +static clib_error_t * +oct_virtio_exit (vlib_main_t *vm) +{ + dao_pal_global_fini (); + return 0; +} + +static clib_error_t * +oct_virtio_plugin_init (vlib_main_t *vm) +{ + dao_lib_logging (); + vec_validate (virtio_port_map, DAO_VIRTIO_DEV_MAX); + vec_validate (oct_virt_thread_data, DAO_PAL_MAX_WORKERS); + vec_validate_aligned (oct_virtio_main, 1, CLIB_CACHE_LINE_BYTES); + return NULL; +} + +static void +oct_virtio_deinit (vlib_main_t *vm, vnet_dev_t *dev) +{ + log_info ("Device unlinitialized\n"); +} + +VLIB_INIT_FUNCTION (oct_virtio_plugin_init); + +VLIB_WORKER_INIT_FUNCTION (oct_virtio_worker_init); + +VLIB_MAIN_LOOP_EXIT_FUNCTION (oct_virtio_exit); + +VNET_DEV_REGISTER_DRIVER (octeon_virtio) = { + .name = "octeon_virtio", + .bus = "virtio", + .device_data_sz = sizeof (oct_virtio_device_t), + .ops = { + .init = oct_virtio_init, + .deinit = oct_virtio_deinit, + .probe = oct_virtio_probe, + }, + .args = oct_virtio_dev_args, +}; + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "OCTEON virtio device", + .default_disabled = 1, +}; diff --git a/src/plugins/dev_octeon/virtio_bus.c b/src/plugins/dev_octeon/virtio_bus.c new file mode 100644 index 000000000..961cdbda4 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_bus.c @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +oct_dev_bus_virtio_device_data_t * +oct_get_bus_virtio_device_data (vnet_dev_t *dev) +{ + return (void *) dev->bus_data; +} + +static int +oct_dev_bus_virtio_device_id_to_virtio_id (u32 *addr, char *str) +{ + unformat_input_t input; + uword rv; + unformat_init_string (&input, str, strlen (str)); + + rv = + unformat (&input, "virtio" VNET_DEV_DEVICE_ID_PREFIX_DELIMITER "%u", addr); + unformat_free (&input); + return rv; +} + +static void * +oct_dev_bus_virtio_get_device_info (vlib_main_t *vm, char *device) +{ + oct_dev_bus_virtio_device_info_t *info; + u32 device_id = 0; + + if (oct_dev_bus_virtio_device_id_to_virtio_id (&device_id, device) == 0) + return 0; + + info = clib_mem_alloc (sizeof (oct_dev_bus_virtio_device_info_t)); + info->virtio_id = device_id; + info->vendor_id = 0x177d; + info->device_id = OCT_VIRTIO_DEVICE_ID; + + return info; +} + +static void +oct_dev_bus_virtio_free_device_info (vlib_main_t *vm, void *dev_info) +{ + clib_mem_free (dev_info); +} + +static vnet_dev_rv_t +oct_dev_bus_virtio_dev_open (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_dev_bus_virtio_device_info_t *info; + oct_dev_bus_virtio_device_data_t *pd = oct_get_bus_virtio_device_data (dev); + + if ((info = oct_dev_bus_virtio_get_device_info (vm, dev->device_id)) == 0) + return VNET_DEV_ERR_INVALID_DEVICE_ID; + + dev->numa_node = 0; + dev->va_dma = 1; + pd->virtio_dev.device_id = info->device_id; + pd->virtio_dev.vendor_id = info->vendor_id; + pd->virtio_dev.virtio_id = info->virtio_id; + + clib_mem_free (info); + + return VNET_DEV_OK; +} + +static void +oct_bus_virtio_dev_close (vlib_main_t *vm, vnet_dev_t *dev) +{ +} + +static u8 * +format_oct_virtio_device_info (u8 *s, va_list *args) +{ + va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + oct_dev_bus_virtio_device_data_t *pdd = oct_get_bus_virtio_device_data (dev); + + s = format (s, "Virtio ID is %u", pdd->virtio_dev.device_id); + + return s; +} + +static u8 * +format_oct_virtio_device_addr (u8 *s, va_list *args) +{ + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + oct_dev_bus_virtio_device_data_t *pdd; + + pdd = oct_get_bus_virtio_device_data (dev); + return format (s, "virtio/%u", pdd->virtio_dev.virtio_id); +} + +VNET_DEV_REGISTER_BUS (virtio) = { + .name = "virtio", + .device_data_size = sizeof (oct_dev_bus_virtio_device_info_t), + .ops = { + .device_open = oct_dev_bus_virtio_dev_open, + .device_close = oct_bus_virtio_dev_close, + .get_device_info = oct_dev_bus_virtio_get_device_info, + .free_device_info = oct_dev_bus_virtio_free_device_info, + .dma_mem_alloc_fn = NULL, + .dma_mem_free_fn = NULL, + .format_device_info = format_oct_virtio_device_info, + .format_device_addr = format_oct_virtio_device_addr, + }, +}; diff --git a/src/plugins/dev_octeon/virtio_bus.h b/src/plugins/dev_octeon/virtio_bus.h new file mode 100644 index 000000000..c039ca96b --- /dev/null +++ b/src/plugins/dev_octeon/virtio_bus.h @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ +#ifndef _VIRTIO_BUS_H_ +#define _VIRTIO_BUS_H_ +#include + +typedef struct +{ + u16 device_id; + u16 vendor_id; + u16 virtio_id; + u16 reserved; +} oct_dev_bus_virtio_device_info_t; + +typedef struct +{ + oct_dev_bus_virtio_device_info_t virtio_dev; +} oct_dev_bus_virtio_device_data_t; + +oct_dev_bus_virtio_device_data_t * +oct_get_bus_virtio_device_data (vnet_dev_t *dev); + +#endif //_VIRTIO_BUS_H diff --git a/src/plugins/dev_octeon/virtio_ctrl.c b/src/plugins/dev_octeon/virtio_ctrl.c new file mode 100644 index 000000000..0c9a5eb37 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_ctrl.c @@ -0,0 +1,370 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include + +#define OCT_VIRTIO_MAX_WRKS 24 +#define OCT_VIRTIO_CHECKSUM_OFFLOAD_MASK 0x3 +#define OCT_VIRTIO_TSO_OFFLOAD_MASK 0xFFFF + +extern oct_virtio_main_t *oct_virtio_main; +extern oct_virtio_port_map_t *virtio_port_map; +extern oct_virtio_per_thread_data_t *oct_virt_thread_data; + +VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { + .class_name = "octeon", + .subclass_name = "virtio_ctl", +}; + +int +oct_virtio_vlib_buffer_free (u16 devid, void *buffs[], u16 nb_buffs) +{ + int i = 0; + u16 hdr_len; + u32 bi[nb_buffs]; + vlib_buffer_t *b[nb_buffs]; + u32 cpu_id = clib_get_current_cpu_id (); + vlib_main_t *vm = vlib_get_first_main (); + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + hdr_len = ptd[cpu_id].q_map[devid].virtio_hdr_sz; + for (i = 0; i < nb_buffs; i++) + b[i] = oct_virt_to_bp (buffs[i], hdr_len); + + vlib_get_buffer_indices (vm, b, bi, nb_buffs); + vlib_buffer_free_no_next (vm, bi, nb_buffs); + + return 0; +} + +int +oct_virtio_vlib_buffer_alloc (u16 devid, void *buffs[], u16 nb_buffs) +{ + int i = 0; + u16 hdr_len; + u16 allocated; + u32 vbuf_idxs[nb_buffs]; + vlib_buffer_t *b[nb_buffs]; + u32 cpu_id = clib_get_current_cpu_id (); + vlib_main_t *vm = vlib_get_first_main (); + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + hdr_len = ptd[cpu_id].q_map[devid].virtio_hdr_sz; + allocated = vlib_buffer_alloc (vm, vbuf_idxs, nb_buffs); + if (allocated != nb_buffs) + { + vlib_buffer_free_no_next (vm, vbuf_idxs, allocated); + return -1; + } + vlib_get_buffers (vm, vbuf_idxs, b, nb_buffs); + + for (i = 0; i < nb_buffs; i++) + buffs[i] = oct_bp_to_virt (b[i], hdr_len); + + return 0; +} + +int +oct_virtio_mac_addr_add (u16 virtio_devid, struct virtio_net_ctrl_mac *mac_tbl, + u8 type) +{ + /* Not supported */ + return 0; +} + +int +oct_virtio_mac_addr_set (u16 virtio_devid, u8 *mac) +{ + /* Not supported */ + return 0; +} + +int +oct_virtio_configure_allmulti (u16 virtio_devid, u8 enable) +{ + /* Not supported */ + return 0; +} + +int +oct_virtio_configure_promisc (u16 virtio_devid, u8 enable) +{ + /* Not supported */ + return 0; +} + +static_always_inline void +oct_virtio_clear_lcore_queue_mapping (u16 virtio_devid) +{ + u32 cpu_id = 0; + oct_virtio_main_t *ovm = oct_virtio_main; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + u64 wrkr_cpu_mask = ovm->wrkr_cpu_mask; + + ovm->netdev_map &= ~(DAO_BIT (virtio_devid)); + while (wrkr_cpu_mask) + { + if (!(wrkr_cpu_mask & (1 << cpu_id))) + { + cpu_id++; + continue; + } + ptd[cpu_id].netdev_map &= ~(DAO_BIT (virtio_devid)); + ptd[cpu_id].q_map[virtio_devid].qmap = 0; + wrkr_cpu_mask &= ~(1 << cpu_id); + cpu_id++; + } + + ovm->netdev_qp_count[virtio_devid] = 0; +} + +static_always_inline u16 +oct_virtio_netdev_hdrlen_get (u16 virtio_devid) +{ + struct virtio_net_hdr vnet_hdr; + u16 virtio_hdr_sz = 0; + u64 feature_bits = 0; + + feature_bits = dao_virtio_netdev_feature_bits_get (virtio_devid); + + if (feature_bits & DAO_BIT_ULL (VIRTIO_NET_F_HASH_REPORT)) + virtio_hdr_sz = offsetof (struct virtio_net_hdr, padding_reserved) + + sizeof (vnet_hdr.padding_reserved); + else + virtio_hdr_sz = offsetof (struct virtio_net_hdr, num_buffers) + + sizeof (vnet_hdr.num_buffers); + return virtio_hdr_sz; +} + +static int +chksum_offload_configure (uint16_t virtio_devid, u64 *tx_offloads, + u64 *rx_offloads) +{ + u64 csum_offload, tso_offload; + + csum_offload = dao_virtio_netdev_feature_bits_get (virtio_devid) & + OCT_VIRTIO_CHECKSUM_OFFLOAD_MASK; + tso_offload = dao_virtio_netdev_feature_bits_get (virtio_devid) & + OCT_VIRTIO_TSO_OFFLOAD_MASK; + + if (csum_offload & DAO_BIT_ULL (VIRTIO_NET_F_CSUM)) + *tx_offloads |= OCT_ETH_TX_OFFLOAD_IPV4_CKSUM; + + if (tso_offload & DAO_BIT_ULL (VIRTIO_NET_F_HOST_TSO4) || + tso_offload & DAO_BIT_ULL (VIRTIO_NET_F_HOST_TSO6)) + { + *tx_offloads |= OCT_ETH_TX_OFFLOAD_TCP_TSO; + log_err ("TSO offload is not supported\n"); + } + + if (csum_offload & DAO_BIT_ULL (VIRTIO_NET_F_GUEST_CSUM)) + *rx_offloads |= OCT_ETH_RX_OFFLOAD_CHECKSUM; + + /** + * We need to configure out interface, but by default, OCTEON interfaces are + * enabled with RX and TX checksum enabled, and currently, we don’t have + * control to enable or disable them. For now, based on these flags, the + * correct flags will be set for the HOST. + */ + return 0; +} + +static_always_inline int +oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) +{ + u32 cpu_id = 0; + u16 virt_rx_q, q_id; + u64 tx_offloads = 0, rx_offloads = 0; + oct_virtio_main_t *ovm = oct_virtio_main; + u64 wrkr_cpu_mask = ovm->wrkr_cpu_mask; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + u16 virtio_hdr_sz = 0; + + virtio_hdr_sz = oct_virtio_netdev_hdrlen_get (virtio_devid); + + chksum_offload_configure (virtio_devid, &tx_offloads, &rx_offloads); + + virt_rx_q = virt_q_count / 2; + q_id = 0; + for (q_id = 0; q_id < virt_rx_q && ovm->wrkr_cpu_mask; q_id++) + { + while (!(wrkr_cpu_mask & DAO_BIT_ULL (cpu_id))) + cpu_id++; + + ptd[cpu_id].q_map[virtio_devid].qmap |= DAO_BIT_ULL (q_id); + CLIB_MEMORY_BARRIER (); + ptd[cpu_id].netdev_map |= DAO_BIT (virtio_devid); + + if (oct_virtio_main->ip4_csum_offload_enable) + { + ptd[cpu_id].intf[virtio_devid].tx_offloads = tx_offloads; + ptd[cpu_id].intf[virtio_devid].rx_offloads = rx_offloads; + } + wrkr_cpu_mask &= ~DAO_BIT_ULL (cpu_id); + cpu_id++; + if (!wrkr_cpu_mask) + { + cpu_id = 0; + wrkr_cpu_mask = ovm->wrkr_cpu_mask; + } + } + + for (cpu_id = 0; cpu_id < DAO_PAL_MAX_WORKERS; cpu_id++) + ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz = virtio_hdr_sz; + + ovm->netdev_qp_count[virtio_devid] = virt_q_count / 2; + CLIB_MEMORY_BARRIER (); + ovm->netdev_map |= DAO_BIT (virtio_devid); + + return 0; +} + +int +oct_virtio_mq_configure (u16 virtio_devid, bool qmap_set) +{ + u16 virt_q_count; + + oct_virtio_clear_lcore_queue_mapping (virtio_devid); + if (!qmap_set) + return 0; + + virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); + log_info ("virtio_dev=%u: virt_q_count=%u\n", virtio_devid, virt_q_count); + if (virt_q_count <= 0 || virt_q_count & 0x1 || + virt_q_count >= (DAO_VIRTIO_MAX_QUEUES - 1)) + { + log_err ("virtio_dev=%d: invalid virt_q_count=%d\n", virtio_devid, + virt_q_count); + return -EIO; + } + + oct_virtio_setup_worker_queue_mapping (virtio_devid, virt_q_count); + + return 0; +} + +int +oct_virito_rss_reta_configure (u16 virtio_devid, + struct virtio_net_ctrl_rss *rss) +{ + u16 virt_q_count; + + oct_virtio_clear_lcore_queue_mapping (virtio_devid); + + if (rss == NULL) + return 0; + + /* Get active virt queue count */ + virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); + + if (virt_q_count <= 0 || virt_q_count & 0x1 || + virt_q_count >= (DAO_VIRTIO_MAX_QUEUES - 1)) + { + log_err ("virtio_dev=%d: invalid virt_q_count=%d\n", virtio_devid, + virt_q_count); + return -EIO; + } + + oct_virtio_setup_worker_queue_mapping (virtio_devid, virt_q_count); + + return 0; +} + +int +oct_virtio_dev_status_cb (u16 virtio_devid, u8 status) +{ + u16 virt_q_count; + + log_debug ("[%s] virtio_dev=%d: status=%s\n", __func__, virtio_devid, + dao_virtio_dev_status_to_str (status)); + + switch (status) + { + case VIRTIO_DEV_RESET: + case VIRTIO_DEV_NEEDS_RESET: + virtio_port_map[virtio_devid].state = 0; + CLIB_MEMORY_BARRIER (); + oct_virtio_clear_lcore_queue_mapping (virtio_devid); + break; + case VIRTIO_DEV_DRIVER_OK: + /* Get active virt queue count */ + virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); + + if (virt_q_count <= 0 || virt_q_count & 0x1 || + virt_q_count >= (DAO_VIRTIO_MAX_QUEUES - 1)) + { + log_err ("virtio_dev=%d: invalid virt_q_count=%d\n", virtio_devid, + virt_q_count); + return -EIO; + } + + oct_virtio_setup_worker_queue_mapping (virtio_devid, virt_q_count); + virtio_port_map[virtio_devid].state = 1; + CLIB_MEMORY_BARRIER (); + break; + default: + break; + } + + return 0; +} + +static_always_inline void +oct_virtio_desc_process (u64 netdev_map, u16 *netdev_qp_count) +{ + u16 dev_id = 0; + + while (netdev_map) + { + if (!(netdev_map & 0x1)) + { + netdev_map >>= 1; + dev_id++; + continue; + } + dao_virtio_net_desc_manage (dev_id, netdev_qp_count[dev_id]); + netdev_map >>= 1; + dev_id++; + } +} + +void +virtio_ctrl_thread_fn (void *args) +{ + vlib_worker_thread_t *w = (vlib_worker_thread_t *) args; + oct_virtio_main_t *ovm = oct_virtio_main; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + u32 cpu_id = clib_get_current_cpu_id (); + + vlib_worker_thread_init (w); + ovm->wrkr_cpu_mask |= DAO_BIT (cpu_id); + ptd->service_core = cpu_id; + /* Wait till Octeon virtio DAO lib init is complete */ + while (!ovm || !ovm->dao_lib_initialized) + CLIB_PAUSE (); + + /* Assign DMA devices per lcore */ + dao_pal_thread_init (cpu_id); + ovm->wrkr_cpu_mask &= ~(DAO_BIT (cpu_id)); + + while (1) + { + /* Process virtio descriptors */ + oct_virtio_desc_process (ovm->netdev_map, ovm->netdev_qp_count); + + /* Flush and submit DMA ops */ + dao_dma_flush_submit (); + } +} + +VLIB_REGISTER_THREAD (virtio_ctrl_thread_reg, static) = { + .name = "virtio-ctrl", + .short_name = "virt-ctl", + .function = virtio_ctrl_thread_fn, + .no_data_structure_clone = 1, +}; diff --git a/src/plugins/dev_octeon/virtio_format.c b/src/plugins/dev_octeon/virtio_format.c new file mode 100644 index 000000000..f21e63488 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_format.c @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include "vlib/pci/pci.h" +#include "vnet/error.h" +#include "vppinfra/error.h" +#include +#include +#include + +u8 * +format_oct_virt_port_status (u8 *s, va_list *args) +{ + return s; +} + +u8 * +format_oct_virt_rx_trace (u8 *s, va_list *args) +{ + va_arg (*args, vlib_main_t *); + va_arg (*args, vlib_node_t *); + oct_virt_rx_trace_t *t = va_arg (*args, oct_virt_rx_trace_t *); + + s = format (s, "octeon-virt-rx: virtio_id %u sw_if_index %u rx_q_map %lu", + t->virtio_id, t->sw_if_index, t->rx_q_map); + return s; +} + +u8 * +format_oct_virt_tx_trace (u8 *s, va_list *args) +{ + va_arg (*args, vlib_main_t *); + va_arg (*args, vlib_node_t *); + oct_virt_tx_trace_t *t = va_arg (*args, oct_virt_tx_trace_t *); + + s = format (s, "octeon-virt-tx: virtio_id %u sw_if_index %u tx_q_map %lu ", + t->virtio_id, t->sw_if_index, t->tx_q_map); + return s; +} diff --git a/src/plugins/dev_octeon/virtio_port.c b/src/plugins/dev_octeon/virtio_port.c new file mode 100644 index 000000000..8814fba8a --- /dev/null +++ b/src/plugins/dev_octeon/virtio_port.c @@ -0,0 +1,171 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define VIRTO_NETDEV_SPEED_NUM_UNKNOWN UINT32_MAX /**< Unknown */ +#define OCT_VIRTIO_NIX_RSS_KEY_LEN 48 + +static u64 vchan_bitmap[2] = { 0 }; +extern oct_virtio_port_map_t *virtio_port_map; +extern oct_virtio_main_t *oct_virtio_main; + +VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { + .class_name = "octeon", + .subclass_name = "virtio_port", +}; + +int +oct_virtio_dma_vchan_id_allocate (void) +{ + int idx; + int pos; + + for (int i = 0; i < DAO_VIRTIO_DEV_MAX; i++) + { + idx = i / 64; + pos = i % 64; + if (!(vchan_bitmap[idx] & (1ULL << pos))) + { + vchan_bitmap[idx] |= (1ULL << pos); + return i; + } + } + return -1; +} + +void +oct_virtio_dma_vchan_id_free (int id) +{ + int idx; + int pos; + + if (id >= 0 && id < DAO_VIRTIO_DEV_MAX) + { + idx = id / 64; + pos = id % 64; + vchan_bitmap[idx] &= ~(1ULL << pos); + } +} + +vnet_dev_rv_t +oct_virtio_port_init (vlib_main_t *vm, vnet_dev_port_t *port) +{ + int rrv; + u16 virtio_devid; + u8 buffer_pool_index; + vlib_buffer_pool_t *bp; + vnet_dev_t *dev = port->dev; + struct dao_virtio_netdev_conf netdev_conf = { 0 }; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + oct_virtio_port_t *ovp = vnet_dev_get_port_data (port); + + buffer_pool_index = + vlib_buffer_pool_get_default_for_numa (vm, dev->numa_node); + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + virtio_devid = ovd->virtio_id; + + netdev_conf.pem_devid = ovd->pem_devid; + netdev_conf.flags |= DAO_VIRTIO_NETDEV_EXTBUF; + netdev_conf.dataroom_size = bp->data_size; + netdev_conf.reta_size = VIRTIO_NET_RSS_RETA_SIZE; + netdev_conf.link_info.status = 0; + netdev_conf.link_info.speed = VIRTO_NETDEV_SPEED_NUM_UNKNOWN; + netdev_conf.link_info.duplex = 0xFF; + netdev_conf.hash_key_size = OCT_VIRTIO_NIX_RSS_KEY_LEN; + netdev_conf.dma_vchan = oct_virtio_dma_vchan_id_allocate (); + netdev_conf.csum_en = oct_virtio_main->ip4_csum_offload_enable; + memcpy (netdev_conf.mac, port->attr.hw_addr.eth_mac, + sizeof (netdev_conf.mac)); + log_debug ("port start: port %u, virtio_id %u, vchan_id %d\n", port->port_id, + virtio_devid, netdev_conf.dma_vchan); + + dao_pal_dma_vchan_setup (virtio_devid, netdev_conf.dma_vchan, NULL); + /* Initialize virtio net device */ + rrv = dao_virtio_netdev_init (virtio_devid, &netdev_conf); + if (rrv) + { + log_err ("[%s] dao_virtio_netdev_init failed \n", __func__); + oct_virtio_dma_vchan_id_free (netdev_conf.dma_vchan); + return VNET_DEV_ERR_INTERNAL; + } + ovp->vchan_id = netdev_conf.dma_vchan; + + return VNET_DEV_OK; +} + +void +oct_virtio_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + oct_virtio_port_t *ovp = vnet_dev_get_port_data (port); + + log_debug ("clear data for virtio id %u\n", ovd->virtio_id); + dao_virtio_netdev_fini (ovd->virtio_id); + oct_virtio_dma_vchan_id_free (ovp->vchan_id); +} + +void +oct_virt_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + vnet_dev_port_state_changes_t changes = {}; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + u16 virtio_devid = ovd->virtio_id; + + if (ovd->status != virtio_port_map[virtio_devid].state) + { + changes.change.link_state = 1; + changes.link_state = virtio_port_map[virtio_devid].state; + ovd->status = virtio_port_map[virtio_devid].state; + } + else + return; + + vnet_dev_port_state_change (vm, port, changes); +} + +vnet_dev_rv_t +oct_virtio_port_start (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + struct dao_virtio_netdev_link_info link_info = { 0 }; + + log_debug ("State up for virtio device %u\n", ovd->virtio_id); + link_info.status = 0x1; + link_info.duplex = 0xFF; + link_info.speed = VIRTO_NETDEV_SPEED_NUM_UNKNOWN; + dao_virtio_netdev_link_sts_update (ovd->virtio_id, &link_info); + + vnet_dev_poll_port_add (vm, port, 0.5, oct_virt_port_poll); + + return VNET_DEV_OK; +} + +void +oct_virtio_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + struct dao_virtio_netdev_link_info link_info = { 0 }; + + log_debug ("[%s] received dev stop port id %d virtio_id %u\n", __func__, + port->port_id, ovd->virtio_id); + + link_info.status = 0x0; + link_info.duplex = 0xFF; + link_info.speed = VIRTO_NETDEV_SPEED_NUM_UNKNOWN; + dao_virtio_netdev_link_sts_update (ovd->virtio_id, &link_info); + vnet_dev_poll_port_remove (vm, port, oct_virt_port_poll); +} diff --git a/src/plugins/dev_octeon/virtio_rx_node.c b/src/plugins/dev_octeon/virtio_rx_node.c new file mode 100644 index 000000000..6aeef9185 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_rx_node.c @@ -0,0 +1,359 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include + +#define OCT_VIRT_LENGTH(h) (h->desc_data[1] & 0x00000000FFFFFFFF) +#define OCT_VIRT_NEXT_FLAG(h) \ + (h->desc_data[1] & DAO_BIT_ULL (VRING_DESC_F_NEXT)) + +extern oct_virtio_main_t *oct_virtio_main; +extern oct_virtio_per_thread_data_t *oct_virt_thread_data; + +static_always_inline u32 +oct_virt_trace_rx_buffers (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t **b, u16 nb_pkts, u32 nb_trace, + u64 rx_q_map, u16 virtio_id, u16 next_index) +{ + int idx = 0; + u32 n_traced = 0; + for (idx = 0; idx < nb_pkts && idx < nb_trace; idx++) + { + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, next_index, b[idx], 0))) + { + oct_virt_rx_trace_t *tr = + vlib_add_trace (vm, node, b[idx], sizeof (*tr)); + tr->rx_q_map = rx_q_map; + tr->virtio_id = virtio_id; + n_traced++; + } + } + + return n_traced; +} + +static_always_inline vlib_buffer_t * +oct_virt_populate_inner_segments (vlib_main_t *vm, vlib_buffer_t *head, + void *p, u32 pool_idx, u16 hdr_len) +{ + vlib_buffer_t *prev, *b; + struct dao_virtio_net_hdr *v_hdr; + + v_hdr = (struct dao_virtio_net_hdr *) p; + b = oct_virt_to_bp (p, hdr_len); + head->total_length_not_including_first_buffer += b->current_length; + prev = b; + while (v_hdr->desc_data[0]) + { + v_hdr = (struct dao_virtio_net_hdr *) v_hdr->desc_data[0]; + b = oct_virt_to_bp ((void *) v_hdr->desc_data[0], hdr_len); + b->current_length = OCT_VIRT_LENGTH (v_hdr) - hdr_len; + b->current_data = 0; + prev->flags |= VLIB_BUFFER_NEXT_PRESENT; + head->total_length_not_including_first_buffer += b->current_length; + prev->buffer_pool_index = pool_idx; + prev->next_buffer = vlib_get_buffer_index (vm, b); + prev = b; + } + head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + + return b; +} + +static_always_inline vlib_buffer_t * +oct_virt_process_chained_packets (vlib_main_t *vm, void **pkts, + u32 *nb_pkts_chain, u32 *n_rx_bytes, + u16 hdr_len) +{ + u32 vhdr_len = sizeof (struct virtio_net_hdr); + struct dao_virtio_net_hdr *v_hdr; + vlib_buffer_t *b, *head, *prev = NULL; + u32 buffer_index = 0, len; + u8 pool_idx = 0; + int idx = 0; + + pool_idx = vlib_buffer_pool_get_default_for_numa (vm, 0); + v_hdr = (struct dao_virtio_net_hdr *) pkts[idx]; + len = OCT_VIRT_LENGTH (v_hdr) - hdr_len; + head = oct_virt_to_bp (pkts[idx], hdr_len); + + /** + * If Host uses linux virtio interface skip first buffer as it contains + * only virtio header details + */ + if (len == vhdr_len) + { + buffer_index = vlib_get_buffer_index (vm, head); + vlib_buffer_free_no_next (vm, &buffer_index, 1); + + idx++; + head = oct_virt_to_bp (pkts[idx], hdr_len); + head->buffer_pool_index = pool_idx; + } + + do + { + v_hdr = (struct dao_virtio_net_hdr *) pkts[idx]; + b = oct_virt_to_bp (pkts[idx], hdr_len); + b->current_length = OCT_VIRT_LENGTH (v_hdr) - hdr_len; + b->current_data = 0; + /* Check for DPU side segmentation */ + if (PREDICT_FALSE ((v_hdr->desc_data[0]))) + b = oct_virt_populate_inner_segments (vm, head, pkts[idx], pool_idx, + hdr_len); + + if (prev) + { + prev->flags |= VLIB_BUFFER_NEXT_PRESENT; + head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + head->total_length_not_including_first_buffer += b->current_length; + prev->buffer_pool_index = pool_idx; + prev->next_buffer = vlib_get_buffer_index (vm, b); + prev = b; + } + prev = b; + idx++; + } + while (OCT_VIRT_NEXT_FLAG (v_hdr)); + + *nb_pkts_chain = idx; + *n_rx_bytes = + head->current_length + head->total_length_not_including_first_buffer; + + return head; +} + +static_always_inline u32 +oct_virtio_process_virtio_packets (vlib_main_t *vm, void **pkts, + vlib_buffer_t **b, u32 *n_rx_pkts, + u32 *to_next, vnet_dev_rx_queue_t *rxq, + u16 hdr_len) +{ + u8 flags = 0; + int idx = 0; + u32 nb_pkts = *n_rx_pkts, next_nb_pkts = 0; + u32 n_rx_bytes = 0, nb_pkts_chain = 0; + struct dao_virtio_net_hdr *v_hdr[4]; + vlib_buffer_template_t bt = vnet_dev_get_rx_queue_if_buffer_template (rxq); + + while (nb_pkts >= 8) + { + v_hdr[0] = (struct dao_virtio_net_hdr *) pkts[idx + 0]; + v_hdr[1] = (struct dao_virtio_net_hdr *) pkts[idx + 1]; + v_hdr[2] = (struct dao_virtio_net_hdr *) pkts[idx + 2]; + v_hdr[3] = (struct dao_virtio_net_hdr *) pkts[idx + 3]; + + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[0]); + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[1]); + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[2]); + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[3]); + + if (PREDICT_FALSE (flags)) + break; + + b[0] = oct_virt_to_bp (pkts[idx + 0], hdr_len); + b[1] = oct_virt_to_bp (pkts[idx + 1], hdr_len); + b[2] = oct_virt_to_bp (pkts[idx + 2], hdr_len); + b[3] = oct_virt_to_bp (pkts[idx + 3], hdr_len); + + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 4], hdr_len)); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 5], hdr_len)); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 6], hdr_len)); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 7], hdr_len)); + + b[0]->template = bt; + b[1]->template = bt; + b[2]->template = bt; + b[3]->template = bt; + + b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]) - hdr_len; + b[1]->current_length = OCT_VIRT_LENGTH (v_hdr[1]) - hdr_len; + b[2]->current_length = OCT_VIRT_LENGTH (v_hdr[2]) - hdr_len; + b[3]->current_length = OCT_VIRT_LENGTH (v_hdr[3]) - hdr_len; + + n_rx_bytes += b[0]->current_length; + n_rx_bytes += b[1]->current_length; + n_rx_bytes += b[2]->current_length; + n_rx_bytes += b[3]->current_length; + + to_next[0] = vlib_get_buffer_index (vm, b[0]); + to_next[1] = vlib_get_buffer_index (vm, b[1]); + to_next[2] = vlib_get_buffer_index (vm, b[2]); + to_next[3] = vlib_get_buffer_index (vm, b[3]); + + b += 4; + idx += 4; + to_next += 4; + nb_pkts -= 4; + next_nb_pkts += 4; + } + + while (nb_pkts) + { + nb_pkts_chain = 1; + v_hdr[0] = (struct dao_virtio_net_hdr *) pkts[idx]; + b[0] = oct_virt_to_bp (pkts[idx], hdr_len); + b[0]->template = bt; + + if (OCT_VIRT_NEXT_FLAG (v_hdr[0])) + b[0] = oct_virt_process_chained_packets ( + vm, &pkts[idx], &nb_pkts_chain, &n_rx_bytes, hdr_len); + else + { + b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]) - hdr_len; + n_rx_bytes += b[0]->current_length; + b[0]->current_data = 0; + } + + to_next[0] = vlib_get_buffer_index (vm, b[0]); + b++; + idx += nb_pkts_chain; + to_next++; + nb_pkts -= nb_pkts_chain; + next_nb_pkts++; + } + + *n_rx_pkts = next_nb_pkts; + + return n_rx_bytes; +} + +static_always_inline uword +oct_virtio_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, vnet_dev_port_t *port, + vnet_dev_rx_queue_t *rxq) +{ + u64 q_map; + u64 rx_q_map; + u32 trace_count; + u32 trace_enabled; + oct_virtio_port_t *ovp; + u32 cpu_id = vm->cpu_id; + void *pkts[VLIB_FRAME_SIZE]; + u32 *to_next, n_left_to_next; + vlib_buffer_t *b[VLIB_FRAME_SIZE]; + vnet_main_t *vnm = vnet_get_main (); + u16 queue, virt_q, virtio_id, hdr_len; + u32 thr_idx = vlib_get_thread_index (); + oct_virtio_main_t *ovm = oct_virtio_main; + u32 n_rx_pkts, n_rx_bytes = 0, rx_pkts_total = 0; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + u16 next_index = vnet_dev_get_rx_queue_if_next_index (rxq); + + trace_enabled = trace_count = vlib_get_trace_count (vm, node); + ovp = vnet_dev_get_port_data (port); + virtio_id = ovp->virtio_id; + + if (PREDICT_FALSE (!ovm || !ovm->dao_lib_initialized)) + return 0; + /* Assign DMA devices per lcore */ + if (PREDICT_FALSE (!ptd[cpu_id].initialized)) + { + dao_pal_thread_init (cpu_id); + dao_pal_dma_lcore_mem2dev_autofree_set (cpu_id, false); + ptd[cpu_id].initialized = 1; + } + + rx_q_map = ptd[cpu_id].q_map[virtio_id].qmap; + q_map = ptd[cpu_id].q_map[virtio_id].qmap; + + if (!(ptd[cpu_id].netdev_map & (DAO_BIT (virtio_id))) || !q_map) + return 0; + + /* Flush and submit DMA ops */ + dao_dma_flush_submit (); + + queue = ptd[cpu_id].q_map[virtio_id].last_rx_q; + hdr_len = ptd[cpu_id].q_map[virtio_id].virtio_hdr_sz; + + while (rx_q_map) + { + if (!(rx_q_map & DAO_BIT (queue))) + goto next; + + rx_q_map &= ~DAO_BIT (queue); + virt_q = (queue << 1) + 1; + + n_rx_pkts = dao_virtio_net_dequeue_burst_ext (virtio_id, virt_q, pkts, + VLIB_FRAME_SIZE); + + if (!n_rx_pkts) + goto next; + + vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next); + + n_rx_bytes += oct_virtio_process_virtio_packets (vm, pkts, b, &n_rx_pkts, + to_next, rxq, hdr_len); + + if (PREDICT_FALSE (trace_count)) + trace_count -= oct_virt_trace_rx_buffers ( + vm, node, b, n_rx_pkts, trace_count, q_map, virtio_id, next_index); + + if (PREDICT_TRUE (next_index == VNET_DEV_ETH_RX_PORT_NEXT_ETH_INPUT)) + { + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + nf = vlib_node_runtime_get_next_frame ( + vm, node, vnet_dev_get_rx_queue_if_sw_if_index (rxq)); + f = vlib_get_frame (vm, nf->frame); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + /** + * We can set the checksum as OK because, in the host checksum + * offload case, OCTEON Tx will perform the checksum computation. In + * the host non-checksum offload case, the host computes the checksum + * and provides it to OCTEON. + */ + f->flags |= ETH_INPUT_FRAME_F_IP4_CKSUM_OK; + + ef = vlib_frame_scalar_args (f); + ef->sw_if_index = vnet_dev_get_rx_queue_if_sw_if_index (rxq); + ef->hw_if_index = vnet_dev_get_rx_queue_if_hw_if_index (rxq); + + vlib_frame_no_append (f); + } + + n_left_to_next -= n_rx_pkts; + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + rx_pkts_total += n_rx_pkts; + + if (rx_pkts_total == VLIB_FRAME_SIZE) + break; + next: + queue = queue + 1; + if (DAO_BIT (queue) > q_map) + queue = 0; + } + + ptd[cpu_id].q_map[virtio_id].last_rx_q = queue; + + if (PREDICT_FALSE (trace_enabled)) + vlib_set_trace_count (vm, node, trace_count); + + vlib_increment_combined_counter ( + vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, + thr_idx, vnet_dev_get_rx_queue_if_hw_if_index (rxq), rx_pkts_total, + n_rx_bytes); + + return rx_pkts_total; +} + +VNET_DEV_NODE_FN (oct_virtio_rx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_rx = 0; + + foreach_vnet_dev_rx_queue_runtime (rxq, node) + { + vnet_dev_port_t *port = rxq->port; + n_rx += oct_virtio_rx_node_inline (vm, node, frame, port, rxq); + } + + return n_rx; +} diff --git a/src/plugins/dev_octeon/virtio_tx_node.c b/src/plugins/dev_octeon/virtio_tx_node.c new file mode 100644 index 000000000..3afebe935 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_tx_node.c @@ -0,0 +1,228 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#define OCT_VIRT_MAX_FRAGS 6 + +extern oct_virtio_main_t *oct_virtio_main; +extern oct_virtio_per_thread_data_t *oct_virt_thread_data; + +static_always_inline void +oct_virt_free_to_vlib (vlib_main_t *vm, vlib_node_runtime_t *node, + void *virt_b[], u16 nb_free, u16 hdr_len) +{ + u16 idx = 0; + vlib_buffer_t *b; + u32 b_index; + + while (idx < nb_free) + { + b = oct_virt_to_bp (virt_b[idx], hdr_len); + b_index = vlib_get_buffer_index (vm, b); + vlib_buffer_free_no_next (vm, &b_index, 1); + idx++; + } + + vlib_error_count (vm, node->node_index, OCT_VIRT_TX_NODE_CTR_ENQUE_FAIL, + nb_free); +} + +static_always_inline u32 +oct_virtio_enqueue (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t **b, u16 nb_pkts, u16 virtio_devid) +{ + u64 rx_offload; + vlib_buffer_t *bp; + bool next_present; + u64 tx_q_map, q_map; + u16 idx = 0, nb_frags = 0; + u32 cpu_id = vm->cpu_id; + u16 nb_pkts_left = nb_pkts, hdr_len; + u16 queue, virt_q, sent = 0, cur_sent = 0; + void *virt_b[VLIB_FRAME_SIZE * OCT_VIRT_MAX_FRAGS]; + struct dao_virtio_net_hdr *v_hdr[4], *head; + struct dao_virtio_net_hdr vhdr_init = { 0 }; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + tx_q_map = ptd[cpu_id].q_map[virtio_devid].qmap; + q_map = ptd[cpu_id].q_map[virtio_devid].qmap; + hdr_len = ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz; + rx_offload = ptd[cpu_id].intf[virtio_devid].rx_offloads; + + /* Packets reaching to tx node means we can assume the checksum is good. */ + if (rx_offload & OCT_ETH_RX_OFFLOAD_CHECKSUM) + vhdr_init.hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID; + + while (nb_pkts >= 8) + { + next_present = b[0]->flags & VLIB_BUFFER_NEXT_PRESENT || + b[1]->flags & VLIB_BUFFER_NEXT_PRESENT || + b[2]->flags & VLIB_BUFFER_NEXT_PRESENT || + b[3]->flags & VLIB_BUFFER_NEXT_PRESENT; + if (PREDICT_FALSE (next_present)) + break; + + v_hdr[0] = oct_bp_to_virt (b[0], hdr_len); + v_hdr[1] = oct_bp_to_virt (b[1], hdr_len); + v_hdr[2] = oct_bp_to_virt (b[2], hdr_len); + v_hdr[3] = oct_bp_to_virt (b[3], hdr_len); + + *v_hdr[0] = vhdr_init; + *v_hdr[1] = vhdr_init; + *v_hdr[2] = vhdr_init; + *v_hdr[3] = vhdr_init; + + virt_b[idx + 0] = (void *) v_hdr[0]; + virt_b[idx + 1] = (void *) v_hdr[1]; + virt_b[idx + 2] = (void *) v_hdr[2]; + virt_b[idx + 3] = (void *) v_hdr[3]; + + clib_prefetch_store (oct_bp_to_virt (b[4], hdr_len)); + clib_prefetch_store (oct_bp_to_virt (b[5], hdr_len)); + clib_prefetch_store (oct_bp_to_virt (b[6], hdr_len)); + clib_prefetch_store (oct_bp_to_virt (b[7], hdr_len)); + + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + + v_hdr[0]->desc_data[1] = b[0]->current_length; + v_hdr[1]->desc_data[1] = b[1]->current_length; + v_hdr[2]->desc_data[1] = b[2]->current_length; + v_hdr[3]->desc_data[1] = b[3]->current_length; + + /* Number of bytes deviates (+/-) from vlib buffer current data */ + v_hdr[0]->desc_data[0] = ~b[0]->current_data + 1; + v_hdr[1]->desc_data[0] = ~b[1]->current_data + 1; + v_hdr[2]->desc_data[0] = ~b[2]->current_data + 1; + v_hdr[3]->desc_data[0] = ~b[3]->current_data + 1; + + v_hdr[0]->hdr.num_buffers = 1; + v_hdr[1]->hdr.num_buffers = 1; + v_hdr[2]->hdr.num_buffers = 1; + v_hdr[3]->hdr.num_buffers = 1; + + b += 4; + idx += 4; + nb_pkts -= 4; + } + + while (nb_pkts) + { + bp = b[0]; + head = oct_bp_to_virt (bp, hdr_len); + do + { + v_hdr[0] = oct_bp_to_virt (bp, hdr_len); + *v_hdr[0] = vhdr_init; + virt_b[idx] = (void *) v_hdr[0]; + v_hdr[0]->desc_data[1] = bp->current_length; + /* Number of bytes deviates (+/-) from vlib buffer current data */ + v_hdr[0]->desc_data[0] = ~bp->current_data + 1; + next_present = bp->flags & VLIB_BUFFER_NEXT_PRESENT; + v_hdr[0]->hdr.num_buffers = 1; + idx++; + nb_frags++; + } + while (next_present && (bp = vlib_get_buffer (vm, bp->next_buffer))); + + head->hdr.num_buffers = nb_frags; + b++; + nb_pkts--; + nb_frags = 0; + } + + queue = ptd[cpu_id].q_map[virtio_devid].last_tx_q; + nb_pkts_left = idx; + + while (tx_q_map && nb_pkts_left) + { + if (!(tx_q_map & DAO_BIT (queue))) + goto next; + + tx_q_map &= ~(DAO_BIT (queue)); + virt_q = queue << 1; + cur_sent = dao_virtio_net_enqueue_burst_ext ( + virtio_devid, virt_q, &virt_b[sent], nb_pkts_left); + nb_pkts_left -= cur_sent; + sent += cur_sent; + + next: + queue = queue + 1; + if (DAO_BIT (queue) > q_map) + queue = 0; + } + + ptd[cpu_id].q_map[virtio_devid].last_tx_q = queue; + + if (PREDICT_FALSE (nb_pkts_left)) + oct_virt_free_to_vlib (vm, node, &virt_b[sent], nb_pkts_left, hdr_len); + + /* Flush and submit DMA ops */ + dao_dma_flush_submit (); + + return sent; +} + +static_always_inline void +oct_virtio_trace_buffers (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_virtio_port_t *ovp, vlib_buffer_t **b, + u16 n_pkts, u16 virtio_id) +{ + u32 i; + u64 tx_q_map; + u32 cpu_id = clib_get_current_cpu_id (); + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + tx_q_map = ptd[cpu_id].q_map[virtio_id].qmap; + + for (i = 0; i < n_pkts; i++) + { + if (!(b[i]->flags & VLIB_BUFFER_IS_TRACED)) + continue; + oct_virt_tx_trace_t *t = vlib_add_trace (vm, node, b[i], sizeof (*t)); + t->virtio_id = virtio_id; + t->sw_if_index = vnet_buffer (b[i])->sw_if_index[VLIB_TX]; + t->tx_q_map = tx_q_map; + } +} + +VNET_DEV_NODE_FN (oct_virtio_tx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_tx_pkts; + u16 virtio_id; + oct_virtio_port_t *ovp; + u32 *from = vlib_frame_vector_args (frame); + u16 n_pkts = frame->n_vectors; + vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers; + vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); + vnet_dev_tx_queue_t *txq = rt->tx_queue; + + if (!txq) + return 0; + + ovp = vnet_dev_get_port_data (txq->port); + virtio_id = ovp->virtio_id; + + vlib_get_buffers (vm, from, b, n_pkts); + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + oct_virtio_trace_buffers (vm, node, ovp, b, n_pkts, virtio_id); + + n_tx_pkts = oct_virtio_enqueue (vm, node, b, n_pkts, virtio_id); + + return n_tx_pkts; +}