report.yaml

$schema: "https://json-schema.org/draft/2020-12/schema"
$id: "https://github.com/WebFuzzing/Commons/blob/master/src/main/resources/wfc/schemas/report.yaml"
title: "Web Fuzzing Commons Report"
description: "Schema Definition for Web Fuzzing Commons Reports"
type: object
properties:
  # REQUIRED
  schemaVersion:
    type: string
    description: "The schema version of WFC needed to use to validate and process this document."
  toolName:
    type: string
    description: "The name of the tool used to create the test cases reported in this document."
  toolVersion:
    type: string
    description: "The version number of the used tool, e.g., 1.0.0."
  creationTime:
    type: string
    format: date-time
    description: "The timestamp of when this report file was created."
  faults:
    $ref: "#/$defs/Faults"
#### Unfortunately, there is no support for oneOf in jsonschema2pojo
#  problemDetails:
#    type: object
#    oneOf:
#      - $ref: "#/$defs/RESTReport"
#      # TODO GraphQL, RPC and Web
  problemDetails:
    type: object
    properties:
      rest:
        $ref: "#/$defs/RESTReport"
      # TODO GraphQL, RPC and Web
      # expressing that only 1 should be present is possible, but super-verbose and convoluted
  totalTests:
    type: integer
    minimum: 0
    description: "The total number of test cases generated by the tool."
  testFilePaths:
    type: array
    items:
      $ref: "#/$defs/TestFilePath"
    uniqueItems: true
    description: "The list of relative paths (compared to this document) of all the generated test suite files."
  testCases:
    description: "Information on each generated test case."
    type: array
    items:
      $ref: "#/$defs/TestCase"
  executionTimeInSeconds:
      type: integer
      minimum: 0
      description: "For how long, in seconds, the tool was running in total."
  #OPTIONAL
  extra:
    description: "Extra, optional coverage information, collected by different tools."
    type: [array, "null"]
    items:
      $ref: "#/$defs/Coverage"

required:
  - "schemaVersion"
  - "toolName"
  - "toolVersion"
  - "creationTime"
  - "faults"
  - "problemDetails"
  - "totalTests"
  - "testFilePaths"
  - "testCases"
  - "executionTimeInSeconds"

$defs:
  OperationId:
    description: "A unique identifier for an operation. For example, in REST, it would be a HTTP endpoint, including 
                  verb, e.g., 'GET:/users/{id}'."
    type: string
  TestCaseId:
    description: "A unique identifier for a test case. It could include its name and file location."
    type: string
  FaultCategoryId:
    description: "A unique identifier for a fault type."
    type: object
    properties:
      code:
        description: "Identifying fault 'code', based on WFC classification."
        type: integer
        anyOf:
          - type: integer
            minimum: 100
            maximum: 104
          - type: integer
            minimum: 200
            maximum: 206
          - type: integer
            minimum: 900
            maximum: 999
      context:
        description: "An optional context for the fault. The same fault type could be manifested in different ways, \
                      and we use this property to differentiate among them."
        type: [string, "null"]
    required: ["code"]
  TestFilePath:
    description: "A relative path used to unique locate a test suite file."
    type: string
  CoveredEndpoint:
    description: "Data-structure to represent which HTTP status code where covered on an endpoint by any of the generated tests."
    type: object
    properties:
      endpointId:
        $ref: "#/$defs/OperationId"
      testCaseId:
        $ref: "#/$defs/TestCaseId"
      httpStatus:
        description: "As in a test case the same endpoint could be called more than once, here we report all of the \
          obtained HTTP status codes. If for any reason a call does not return any response (e.g., the TCP connection \ 
          does timeout), then this HTTP status array would be either null or empty."
        type: [array,"null"]
        items:
          $ref: "#/$defs/HttpStatus"
        minItems: 0
        uniqueItems: true
    required: ["endpointId","testCaseId","httpStatus"]
  HttpStatus:
    type: [integer,"null"]
    minimum: 0
    maximum: 599
  FoundFault:
    description: "Data-structure to represent found faults, based on operations (e.g., HTTP endpoints in REST, and methods \
                  in GraphQL and RPC) and which tests find faults in them."
    type: object
    properties:
      operationId:
        $ref: "#/$defs/OperationId"
      testCaseId:
        $ref: "#/$defs/TestCaseId"
      faultCategories:
        type: array
        items:
          $ref: "#/$defs/FaultCategoryId"
        minItems: 1
        uniqueItems: true
    required: ["endpointId","testCaseId","faultCategories"]

  RESTReport:
    type: object
    properties:
      outputHttpCalls:
        description: "Total number of HTTP calls made in all the generated test cases given as output. \ 
                      A test case could contain several HTTP calls, \
                      e.g., a POST followed by a GET and then a DELETE."
        type: integer
        minimum: 0
      evaluatedHttpCalls:
        description: "Total number of all HTTP calls made during the whole fuzzing session. \
                      If the fuzzing was left running for hours, millions of HTTP could have been made.
                      The output generated tests will only contain a tiny subset of these evaluated calls."
        type: integer
        minimum: 0
      endpointIds:
        description: "Unique ids of all the endpoints in the tested API."
        type: array
        items:
          $ref: "#/$defs/OperationId"
        uniqueItems: true
      coveredHttpStatus:
        description: "List of which HTTP status codes were covered, based on endpoints."
        type: array
        items:
          $ref: "#/$defs/CoveredEndpoint"
    required: ["outputHttpCalls","evaluatedHttpCalls","endpointIds","coveredHttpStatus"]

  TestCase:
    type: object
    properties:
      id:
        $ref: "#/$defs/TestCaseId"
      filePath:
        $ref: "#/$defs/TestFilePath"
      name:
        description: "The name of the test case, as it appears in the generated test file."
        type: string
      startLine:
        description: "The line number in the generated test suite file where the code of this test case starts."
        type: integer
        minimum: 0
      endLine:
        description: "The line number in the generated test suite file where the code of this test case ends."
        type: integer
        minimum: 0

  Faults:
    type: object
    properties:
      totalNumber:
        description: "The total number of potential faults identified in the generated test suites. \
                      Each fault is uniquely identified with its 'code' category and 'context'. \
                      Note that different tests can detect the same fault, and a test case can detect several different faults."
        type: integer
        minimum: 0
      foundFaults:
        description: "Information on all the identified potential faults."
        type: array
        items:
          $ref: "#/$defs/FoundFault"
    required: ["totalNumber","foundFaults"]

  Coverage:
    type: object
    properties:
      toolName:
        description: "The name of the tool used to collect and compute the coverage criteria."
        type: string
      criteria:
        type: array
        items:
          $ref: "#/$defs/CoverageCriterion"
    required: ["toolName","criteria"]

  CoverageCriterion:
    type: object
    properties:
      name:
        description: "The name of this coverage criterion."
        type: string
      covered:
        description: "The number of testing targets for this criterion that were covered."
        type: integer
        minimum: 0
      total:
        description: "Optional number of all testing targets for this criterion. For some criteria, this number can be unknown."
        type: [integer, "null"]
        minimum: 0
    required: ["name","covered"]