EvalFactsheets/schema.json at main · evaleval/EvalFactsheets · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "Evaluation Factsheet",
  "description": "A schema for evaluation factsheets describing benchmarks and evaluations.",
  "type": "object",
  "properties": {
    "title": {
      "type": "string",
      "description": "The official name of the evaluation or benchmark."
    },
    "subtitle": {
      "type": "string",
      "description": "A concise description of what the evaluation measures."
    },
    "authors": {
      "type": "string",
      "description": "Who made this evaluation (Org/University/Company/Lab/Authors)."
    },
    "link": {
      "type": "string",
      "format": "uri",
      "description": "URL to the paper or documentation describing the evaluation."
    },
    "code_link": {
      "type": "string",
      "format": "uri",
      "description": "URL to the repository containing the evaluation implementation."
    },
    "date": {
      "type": "string",
      "format": "date",
      "description": "Publication or release date of the evaluation."
    },
    "purpose": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "The primary intended use case(s) for this evaluation."
    },
    "principlesTested": {
      "type": "string",
      "description": "Principles or capabilities being tested."
    },
    "functionalProps": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Functional properties of the evaluation."
    },
    "inputModality": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Input modalities (e.g., Text, Image)."
    },
    "outputModality": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Output modalities (e.g., Text, Image)."
    },
    "inputSource": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Source of the input data."
    },
    "outputSource": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Source of the output data (e.g., Model generated)."
    },
    "size": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Size of the evaluation dataset."
    },
    "splits": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Dataset splits (e.g., Train, Test, Validation)."
    },
    "design": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Design of the evaluation."
    },
    "judge": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Who or what judges the output."
    },
    "protocol": {
      "type": "string",
      "description": "Description of the evaluation protocol."
    },
    "modelAccess": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Level of model access required."
    },
    "hasHeldout": {
      "type": "boolean",
      "description": "Whether there is a held-out test set."
    },
    "heldoutDetails": {
      "type": "string",
      "description": "Details about the held-out test set."
    },
    "alignmentValidation": {
      "type": "string",
      "description": "Validation of the alignment measurement."
    },
    "isValid": {
      "type": "string",
      "description": "Construct validity score or status."
    },
    "baselineModels": {
      "type": "string",
      "description": "List of baseline models and their performance."
    },
    "robustnessMeasures": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Robustness measures applied."
    },
    "knownLimitations": {
      "type": "string",
      "description": "Known sensitivities and limitations."
    },
    "benchmarksList": {
      "type": "string",
      "description": "List of similar evaluations or benchmarks."
    }
  },
  "required": [
    "title",
    "subtitle",
    "authors",
    "date"
  ]
}