openbmc
diff --git a/‎src/nvidia-gpu/NvidiaGpuMctpVdm.cpp‎
Lines changed: 97 additions & 3 deletions b/‎src/nvidia-gpu/NvidiaGpuMctpVdm.cpp‎
Lines changed: 97 additions & 3 deletions
diff --git a/‎src/nvidia-gpu/NvidiaGpuMctpVdm.hpp‎
Lines changed: 20 additions & 0 deletions b/‎src/nvidia-gpu/NvidiaGpuMctpVdm.hpp‎
Lines changed: 20 additions & 0 deletions
diff --git a/‎src/nvidia-gpu/NvidiaPcieDevice.cpp‎
Lines changed: 94 additions & 0 deletions b/‎src/nvidia-gpu/NvidiaPcieDevice.cpp‎
Lines changed: 94 additions & 0 deletions
diff --git a/‎src/nvidia-gpu/NvidiaPcieDevice.hpp‎
Lines changed: 21 additions & 1 deletion b/‎src/nvidia-gpu/NvidiaPcieDevice.hpp‎
Lines changed: 21 additions & 1 deletion
@@ -12,6 +12,7 @@
 #include <cerrno>
 #include <cstddef>
 #include <cstdint>
+#include <cstring>
 #include <span>
 #include <vector>
 
@@ -564,12 +565,105 @@ int decodeQueryScalarGroupTelemetryV2Response(
         telemetryValues.resize(numTelemetryValues);
     }
 
-    const auto* telemetryDataPtr = reinterpret_cast<const uint32_t*>(
-        buf.data() + sizeof(ocp::accelerator_management::CommonResponse));
+    const auto* telemetryDataPtr =
+        buf.data() + sizeof(ocp::accelerator_management::CommonResponse);
 
     for (size_t i = 0; i < numTelemetryValues; i++)
     {
-        telemetryValues[i] = le32toh(telemetryDataPtr[i]);
+        std::memcpy(&telemetryValues[i],
+                    telemetryDataPtr + i * sizeof(uint32_t), sizeof(uint32_t));
+
+        telemetryValues[i] = le32toh(telemetryValues[i]);
+    }
+
+    return 0;
+}
+
+int encodeListPciePortsRequest(uint8_t instanceId, std::span<uint8_t> buf)
+{
+    if (buf.size() < sizeof(ocp::accelerator_management::CommonRequest))
+    {
+        return EINVAL;
+    }
+
+    auto* msg = reinterpret_cast<ocp::accelerator_management::CommonRequest*>(
+        buf.data());
+
+    ocp::accelerator_management::BindingPciVidInfo header{};
+    header.ocp_accelerator_management_msg_type =
+        static_cast<uint8_t>(ocp::accelerator_management::MessageType::REQUEST);
+    header.instance_id = instanceId &
+                         ocp::accelerator_management::instanceIdBitMask;
+    header.msg_type = static_cast<uint8_t>(MessageType::PCIE_LINK);
+
+    auto rc = packHeader(header, msg->msgHdr.hdr);
+
+    if (rc != 0)
+    {
+        return rc;
+    }
+
+    msg->command = static_cast<uint8_t>(PcieLinkCommands::ListPCIePorts);
+    msg->data_size = 0;
+
+    return 0;
+}
+
+int decodeListPciePortsResponse(
+    std::span<const uint8_t> buf,
+    ocp::accelerator_management::CompletionCode& cc, uint16_t& reasonCode,
+    uint16_t& numUpstreamPorts, std::vector<uint8_t>& numDownstreamPorts)
+{
+    auto rc =
+        ocp::accelerator_management::decodeReasonCodeAndCC(buf, cc, reasonCode);
+
+    if (rc != 0 || cc != ocp::accelerator_management::CompletionCode::SUCCESS)
+    {
+        return rc;
+    }
+
+    if (buf.size() < sizeof(ListPCIePortsResponse))
+    {
+        return EINVAL;
+    }
+
+    const auto* response =
+        reinterpret_cast<const ListPCIePortsResponse*>(buf.data());
+
+    const uint16_t dataSize = le16toh(response->hdr.data_size);
+
+    if (dataSize < sizeof(uint16_t))
+    {
+        return EINVAL;
+    }
+
+    uint16_t upstreamPorts = le16toh(response->numUpstreamPorts);
+
+    numUpstreamPorts = 0;
+    numDownstreamPorts.clear();
+    numDownstreamPorts.reserve(upstreamPorts);
+
+    size_t offset = sizeof(ListPCIePortsResponse);
+
+    for (size_t i = 0; i < upstreamPorts; i++)
+    {
+        if (offset + sizeof(ListPCIePortsDownstreamPortsData) > buf.size())
+        {
+            return EINVAL;
+        }
+
+        const auto* downstreamPortData =
+            reinterpret_cast<const ListPCIePortsDownstreamPortsData*>(
+                buf.data() + offset);
+
+        // Count only external upstream ports
+        if (downstreamPortData->isInternal == 0)
+        {
+            ++numUpstreamPorts;
+            numDownstreamPorts.push_back(downstreamPortData->count);
+        }
+
+        offset += sizeof(ListPCIePortsDownstreamPortsData);
     }
 
     return 0;
 
@@ -48,6 +48,7 @@ enum class PlatformEnvironmentalCommands : uint8_t
 
 enum class PcieLinkCommands : uint8_t
 {
+    ListPCIePorts = 0x07,
     QueryScalarGroupTelemetryV2 = 0x24,
 };
 
@@ -176,6 +177,18 @@ struct GetVoltageResponse
     uint32_t voltage;
 } __attribute__((packed));
 
+struct ListPCIePortsResponse
+{
+    ocp::accelerator_management::CommonResponse hdr;
+    uint16_t numUpstreamPorts;
+} __attribute__((packed));
+
+struct ListPCIePortsDownstreamPortsData
+{
+    uint8_t isInternal;
+    uint8_t count;
+} __attribute__((packed));
+
 struct GetInventoryInformationRequest
 {
     ocp::accelerator_management::CommonRequest hdr;
@@ -255,4 +268,11 @@ int decodeQueryScalarGroupTelemetryV2Response(
     ocp::accelerator_management::CompletionCode& cc, uint16_t& reasonCode,
     size_t& numTelemetryValues, std::vector<uint32_t>& telemetryValues);
 
+int encodeListPciePortsRequest(uint8_t instanceId, std::span<uint8_t> buf);
+
+int decodeListPciePortsResponse(
+    std::span<const uint8_t> buf,
+    ocp::accelerator_management::CompletionCode& cc, uint16_t& reasonCode,
+    uint16_t& numUpstreamPorts, std::vector<uint8_t>& numDownstreamPorts);
+
 } // namespace gpu
@@ -6,19 +6,25 @@
 #include "NvidiaPcieDevice.hpp"
 
 #include "NvidiaDeviceDiscovery.hpp"
+#include "NvidiaGpuMctpVdm.hpp"
 #include "NvidiaPcieInterface.hpp"
+#include "NvidiaPciePort.hpp"
 #include "Utils.hpp"
 
 #include <MctpRequester.hpp>
+#include <OcpMctpVdm.hpp>
 #include <boost/asio/io_context.hpp>
 #include <phosphor-logging/lg2.hpp>
 #include <sdbusplus/asio/connection.hpp>
 #include <sdbusplus/asio/object_server.hpp>
 
 #include <chrono>
 #include <cstdint>
+#include <format>
 #include <memory>
+#include <span>
 #include <string>
+#include <system_error>
 
 PcieDevice::PcieDevice(const SensorConfigs& configs, const std::string& name,
                        const std::string& path,
@@ -34,6 +40,65 @@ PcieDevice::PcieDevice(const SensorConfigs& configs, const std::string& name,
 
 void PcieDevice::init()
 {
+    getPciePortCounts();
+}
+
+void PcieDevice::getPciePortCounts()
+{
+    const int rc = gpu::encodeListPciePortsRequest(0, getPciePortCountsRequest);
+
+    if (rc != 0)
+    {
+        lg2::error(
+            "Error updating PCIe Port Counts: encode failed, rc={RC}, EID={EID}",
+            "RC", rc, "EID", eid);
+        return;
+    }
+
+    mctpRequester.sendRecvMsg(
+        eid, getPciePortCountsRequest,
+        [weak{weak_from_this()}](const std::error_code& ec,
+                                 std::span<const uint8_t> buffer) {
+            std::shared_ptr<PcieDevice> self = weak.lock();
+            if (!self)
+            {
+                lg2::error("Invalid reference to PcieDevice");
+                return;
+            }
+            self->processPciePortCountsResponse(ec, buffer);
+        });
+}
+
+void PcieDevice::processPciePortCountsResponse(
+    const std::error_code& ec, std::span<const uint8_t> response)
+{
+    if (ec)
+    {
+        lg2::error(
+            "Error processing PCIe Port Counts response: sending message over MCTP failed, rc={RC}, EID={EID}",
+            "RC", ec.message(), "EID", eid);
+        return;
+    }
+
+    ocp::accelerator_management::CompletionCode cc{};
+    uint16_t reasonCode = 0;
+
+    const int rc = gpu::decodeListPciePortsResponse(
+        response, cc, reasonCode, pcieDeviceInfo.numUpstreamPorts,
+        pcieDeviceInfo.numDownstreamPorts);
+
+    if (rc != 0 || cc != ocp::accelerator_management::CompletionCode::SUCCESS)
+    {
+        lg2::error(
+            "Error processing PCIe Port Counts response: decode failed, rc={RC}, cc={CC}, reasonCode={RESC}, EID={EID}",
+            "RC", rc, "CC", static_cast<uint8_t>(cc), "RESC", reasonCode, "EID",
+            eid);
+        return;
+    }
+
+    lg2::info("PCIe Device with eid {EID} has {UP} upstream ports.", "EID", eid,
+              "UP", pcieDeviceInfo.numUpstreamPorts);
+
     makeSensors();
 }
 
@@ -42,6 +107,30 @@ void PcieDevice::makeSensors()
     pcieInterface = std::make_shared<NvidiaPcieInterface>(
         conn, mctpRequester, name, path, eid, objectServer);
 
+    uint64_t downstreamPortIndex = 0;
+
+    for (uint64_t i = 0; i < pcieDeviceInfo.numUpstreamPorts; ++i)
+    {
+        const std::string portName = std::format("UP_{}", i);
+
+        pciePorts.emplace_back(std::make_shared<NvidiaPciePortInfo>(
+            conn, mctpRequester, portName, name, path, eid,
+            gpu::PciePortType::UPSTREAM, i, i, objectServer));
+
+        for (uint64_t j = 0; j < pcieDeviceInfo.numDownstreamPorts[i]; ++j)
+        {
+            const std::string portName =
+                std::format("DOWN_{}", downstreamPortIndex);
+
+            pciePorts.emplace_back(std::make_shared<NvidiaPciePortInfo>(
+                conn, mctpRequester, portName, name, path, eid,
+                gpu::PciePortType::DOWNSTREAM, i, downstreamPortIndex,
+                objectServer));
+
+            ++downstreamPortIndex;
+        }
+    }
+
     lg2::info("Added PCIe {NAME} Sensors with chassis path: {PATH}.", "NAME",
               name, "PATH", path);
 
@@ -52,6 +141,11 @@ void PcieDevice::read()
 {
     pcieInterface->update();
 
+    for (auto& port : pciePorts)
+    {
+        port->update();
+    }
+
     waitTimer.expires_after(std::chrono::milliseconds(sensorPollMs));
     waitTimer.async_wait([this](const boost::system::error_code& ec) {
         if (ec)
 
@@ -9,6 +9,7 @@
 #include "NvidiaDeviceDiscovery.hpp"
 #include "NvidiaPcieInterface.hpp"
 
+#include <NvidiaPciePort.hpp>
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <sdbusplus/asio/connection.hpp>
@@ -18,11 +19,18 @@
 #include <cstdint>
 #include <memory>
 #include <string>
+#include <vector>
 
 constexpr const char* pcieDevicePathPrefix =
     "/xyz/openbmc_project/inventory/pcie_devices/";
 
-class PcieDevice
+struct PcieDeviceInfo
+{
+    uint16_t numUpstreamPorts{};
+    std::vector<uint8_t> numDownstreamPorts;
+};
+
+class PcieDevice : public std::enable_shared_from_this<PcieDevice>
 {
   public:
     PcieDevice(const SensorConfigs& configs, const std::string& name,
@@ -44,6 +52,13 @@ class PcieDevice
 
     void read();
 
+    void getPciePortCounts();
+
+    void processPciePortCountsResponse(const std::error_code& ec,
+                                       std::span<const uint8_t> response);
+
+    PcieDeviceInfo pcieDeviceInfo;
+
     uint8_t eid{};
 
     std::chrono::milliseconds sensorPollMs;
@@ -62,5 +77,10 @@ class PcieDevice
 
     std::string path;
 
+    std::array<uint8_t, sizeof(ocp::accelerator_management::CommonRequest)>
+        getPciePortCountsRequest{};
+
     std::shared_ptr<NvidiaPcieInterface> pcieInterface;
+
+    std::vector<std::shared_ptr<NvidiaPciePortInfo>> pciePorts;
 };