not declared faults

Author: omursahin

web-report/src/assets/info.json

@@ -10,6 +10,8 @@
   "numberOfFaultsPerCode": "Number of faults detected for each code.",
   "distributionOfEndpointsPerCode": "Faulty endpoints per code (Affected/Total Endpoints).",
   "distributionTooltip": "{operationCount} {endpointText} {code} error code out of {totalEndpoints} endpoints.",
+  "undeclaredOperationsTooltip": "In addition, {undeclaredCount} operation(s) with this fault are not declared in the API schema, and so are not counted in the endpoint totals.",
+  "notInSchema": "Faults were detected on this operation, but it is not declared in the API schema. It is not counted in the endpoint statistics.",
   "generatedTestFiles": "Number of generated test files.",
   "generatedTestCases": "Number of generated test cases.",
   "totalFaults": "Total number of faults detected in the API.",

web-report/src/components/EndpointAccordion.tsx

@@ -3,6 +3,7 @@ import {Badge} from "@/components/ui/badge.tsx";
 import React, {useState} from "react";
 import {TestCases} from "@/components/TestCases.tsx";
 import {getColor, getHoverColor} from "@/lib/utils";
+import info from "@/assets/info.json";
 
 interface IStatusType {
     code: number | string;
@@ -12,6 +13,7 @@ interface IStatusType {
 export interface IEndpointAccordionProps {
     endpoint: string;
     value: string;
+    declared?: boolean;
     statusCodes: IStatusType[];
     faults: IStatusType[];
     addTestTab: (value: string, event: React.MouseEvent<HTMLElement>) => void;
@@ -20,6 +22,7 @@ export interface IEndpointAccordionProps {
 export const EndpointAccordion: React.FC<IEndpointAccordionProps> = ({
                                                                         endpoint,
                                                                         value,
+                                                                        declared = true,
                                                                         statusCodes,
                                                                         faults,
                                                                         addTestTab
@@ -73,9 +76,14 @@ export const EndpointAccordion: React.FC<IEndpointAccordionProps> = ({
     const faultColors = ["bg-red-300", "bg-red-500", "bg-red-700"];
     return (
         <AccordionItem value={value} className="border-2 border-black mb-4 overflow-hidden" data-testid={endpoint}>
-            <AccordionTrigger className="bg-blue-100 px-3 sm:px-4 py-3 text-sm sm:text-lg font-bold hover:no-underline hover:bg-blue-200">
+            <AccordionTrigger className={`px-3 sm:px-4 py-3 text-sm sm:text-lg font-bold hover:no-underline ${declared ? "bg-blue-100 hover:bg-blue-200" : "bg-amber-100 hover:bg-amber-200"}`}>
                 <div className="flex-1 font-mono break-all text-left">{endpoint}</div>
                 <div className="flex flex-wrap justify-end gap-1 sm:gap-2 mr-2 sm:mr-4">
+                    {!declared && (
+                        <Badge className="bg-amber-500 font-mono text-xs" title={info.notInSchema}>
+                            NOT IN SCHEMA
+                        </Badge>
+                    )}
                     {sortedStatusCodes.map((code, idx) => (
                         <Badge key={`_${idx}`} className={`${getColor(code.code, true, false)} font-mono text-xs`}>
                             {code.code == -1 ? "NO-RESPONSE" : `H${code.code}`}

web-report/src/components/FaultsComponent.tsx

@@ -11,9 +11,10 @@ import faults from "../../../src/main/resources/wfc/faults/fault_categories.json
 
 export const FaultsComponent: React.FC<Faults> = ({totalNumber, foundFaults}) => {
     const {data} = useAppContext();
-    const totalEndpointNumber = data?.problemDetails.rest?.endpointIds.length;
+    const endpointIds = data?.problemDetails.rest?.endpointIds;
+    const totalEndpointNumber = endpointIds?.length;
 
-    const faultCounts = getFaultCounts(foundFaults);
+    const faultCounts = getFaultCounts(foundFaults, endpointIds);
     const [isModalOpen, setIsModalOpen] = useState(false)
     const [currentStatus, setCurrentStatus] = useState(-1);
 
@@ -80,8 +81,14 @@ export const FaultsComponent: React.FC<Faults> = ({totalNumber, foundFaults}) =>
                                         endpointText: fault.operationCount > 1 ? "endpoints have" : "endpoint has",
                                         code: fault.code,
                                         totalEndpoints:totalEndpointNumber ? totalEndpointNumber : 0
-                                    })}>
-                                    <div>{fault.operationCount}/{totalEndpointNumber}</div>
+                                    }) + (fault.undeclaredOperationCount > 0
+                                        ? " " + getText(info.undeclaredOperationsTooltip, {undeclaredCount: fault.undeclaredOperationCount})
+                                        : "")}>
+                                    <div>
+                                        {fault.operationCount}/{totalEndpointNumber}
+                                        {fault.undeclaredOperationCount > 0 &&
+                                            <span className="text-amber-600 font-bold"> +{fault.undeclaredOperationCount}</span>}
+                                    </div>
                                 </ReportTooltip>
                                 <div className="col-span-2 text-center font-bold">{fault.count}</div>
                             </div>

web-report/src/lib/utils.tsx

@@ -199,7 +199,7 @@ export const calculateAllStatusCounts = (coveredHttpStatus: CoveredEndpoint[], e
     return allStatusCounts;
 }
 
-export const getFaultCounts = (foundFaults: FoundFault[]) => {
+export const getFaultCounts = (foundFaults: FoundFault[], endpointIds?: string[]) => {
     const faultCounts = new Map();
     // A fault defines a unique operationId, code, and context. To define a unique fault, we can use a combination of these three properties.
 
@@ -222,11 +222,17 @@ export const getFaultCounts = (foundFaults: FoundFault[]) => {
 
     return Array.from(uniqueCodes).map(code => {
         const faultsWithCode = uniqueFaults.filter(fault => fault.code === code);
-        const uniqueOperationCounts = new Set(faultsWithCode.map(fault => fault.operationId)).size;
+        const uniqueOperations = new Set(faultsWithCode.map(fault => fault.operationId));
+        // Operations not declared in the schema must not skew the endpoint ratio,
+        // so they are reported separately instead of being counted in operationCount
+        const undeclaredOperationCount = endpointIds
+            ? [...uniqueOperations].filter(operation => !endpointIds.includes(operation)).length
+            : 0;
         return {
             code: code,
             count: faultsWithCode.length,
-            operationCount: uniqueOperationCounts,
+            operationCount: uniqueOperations.size - undeclaredOperationCount,
+            undeclaredOperationCount: undeclaredOperationCount,
         }
     }).sort((a, b) => a.code - b.code);
 }
@@ -273,6 +279,9 @@ export const getLanguage = (fileName: string) => {
 
 export interface ITransformedReport {
     endpoint: string;
+    // false when the operation is not declared in the API schema (endpointIds),
+    // but faults were still detected on it (e.g. an undocumented OPTIONS handler)
+    declared: boolean;
     faults: {
         code: number;
         testCases: string[];
@@ -294,6 +303,7 @@ export const transformWebFuzzingReport = (original: WebFuzzingCommonsReport | nu
     original.problemDetails.rest?.endpointIds.forEach(endpoint => {
         endpointMap.set(endpoint, {
             endpoint,
+            declared: true,
             httpStatusCodes: [],
             faults: []
         });
@@ -305,7 +315,12 @@ export const transformWebFuzzingReport = (original: WebFuzzingCommonsReport | nu
         }
 
         if (!endpointMap.has(fault.operationId)) {
-            console.log(`Endpoint ${fault.operationId} not found in endpointIds`);
+            endpointMap.set(fault.operationId, {
+                endpoint: fault.operationId,
+                declared: false,
+                httpStatusCodes: [],
+                faults: []
+            });
         }
 
         const endpointData = endpointMap.get(fault.operationId);

web-report/src/pages/Endpoints.tsx

@@ -14,20 +14,33 @@ export const Endpoints: React.FC<IProps> = ({addTestTab, openEndpoint, setOpenEn
 
     const {transformedReport, filteredEndpoints, statusFilters, setStatusFilters} = useAppContext();
 
+    const declaredEndpoints = transformedReport.filter(endpoint => endpoint.declared);
+    const filteredDeclaredEndpoints = filteredEndpoints.filter(endpoint => endpoint.declared);
+    const undeclaredEndpoints = transformedReport.filter(endpoint => !endpoint.declared);
+    const filteredUndeclaredEndpoints = filteredEndpoints.filter(endpoint => !endpoint.declared);
+
     return (
         <div className="border-2 border-black p-3 sm:p-6 rounded-none">
             <StatusCodeFilters data={transformedReport} filters={statusFilters} onFiltersChange={setStatusFilters}/>
             <div className="flex items-center mb-2">
                 <h3 className="text-sm font-medium text-gray-700 mr-3"># Endpoints:</h3>
                 <div className="flex flex-wrap gap-2 font-bold font-mono">
-                    <p className="text-black-400">{filteredEndpoints.length}</p> / <p className="text-red-400">{transformedReport.length}</p>
+                    <p className="text-black-400">{filteredDeclaredEndpoints.length}</p> / <p className="text-red-400">{declaredEndpoints.length}</p>
                 </div>
             </div>
+            {undeclaredEndpoints.length > 0 &&
+                <div className="flex items-center mb-2">
+                    <h3 className="text-sm font-medium text-gray-700 mr-3"># Faulty operations not in schema:</h3>
+                    <div className="flex flex-wrap gap-2 font-bold font-mono">
+                        <p className="text-black-400">{filteredUndeclaredEndpoints.length}</p> / <p className="text-amber-600">{undeclaredEndpoints.length}</p>
+                    </div>
+                </div>
+            }
             <Accordion type="single" collapsible value={openEndpoint} onValueChange={setOpenEndpoint} className="w-full">
                 {
                     filteredEndpoints.map((item, index) => (
                         <EndpointAccordion data-testid="endpoint" key={index} value={item.endpoint}
-                                           endpoint={item.endpoint}
+                                           endpoint={item.endpoint} declared={item.declared}
                                            statusCodes={item.httpStatusCodes} faults={item.faults}
                                            addTestTab={addTestTab}/>
                     ))