Skip to content

Commit 53ca65d

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: Update multimodal evaluation (content_map_instance), rubric generation (rubric_based_instance, etc) and raw_output(raw_output, custom_output, etc) proto change in online eval API
docs: A comment for field `autorater_config` in message `.google.cloud.aiplatform.v1beta1.EvaluateDatasetRequest` is changed docs: A comment for field `gcs_source` in message `.google.cloud.aiplatform.v1beta1.EvaluationDataset` is changed PiperOrigin-RevId: 739314382
1 parent d2dcfcf commit 53ca65d

1 file changed

Lines changed: 131 additions & 2 deletions

File tree

‎google/cloud/aiplatform/v1beta1/evaluation_service.proto‎

Lines changed: 131 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import "google/api/annotations.proto";
2020
import "google/api/client.proto";
2121
import "google/api/field_behavior.proto";
2222
import "google/api/resource.proto";
23+
import "google/cloud/aiplatform/v1beta1/content.proto";
2324
import "google/cloud/aiplatform/v1beta1/io.proto";
2425
import "google/cloud/aiplatform/v1beta1/operation.proto";
2526
import "google/longrunning/operations.proto";
@@ -118,7 +119,9 @@ message EvaluateDatasetRequest {
118119
// Required. Config for evaluation output.
119120
OutputConfig output_config = 4 [(google.api.field_behavior) = REQUIRED];
120121

121-
// Optional. Autorater config used for evaluation.
122+
// Optional. Autorater config used for evaluation. Currently only publisher
123+
// Gemini models are supported. Format:
124+
// `projects/{PROJECT}/locations/{LOCATION}/publishers/google/models/{MODEL}.`
122125
AutoraterConfig autorater_config = 5 [(google.api.field_behavior) = OPTIONAL];
123126
}
124127

@@ -196,7 +199,8 @@ message Metric {
196199
message EvaluationDataset {
197200
// The source of the dataset.
198201
oneof source {
199-
// Cloud storage source holds the dataset.
202+
// Cloud storage source holds the dataset. Currently only one Cloud Storage
203+
// file path is supported.
200204
GcsSource gcs_source = 1;
201205

202206
// BigQuery source holds the dataset.
@@ -335,6 +339,10 @@ message EvaluateInstancesRequest {
335339

336340
// Input for trajectory single tool use metric.
337341
TrajectorySingleToolUseInput trajectory_single_tool_use_input = 39;
342+
343+
// Rubric Based Instruction Following metric.
344+
RubricBasedInstructionFollowingInput
345+
rubric_based_instruction_following_input = 40;
338346
}
339347

340348
// Required. The resource name of the Location to evaluate the instances.
@@ -460,6 +468,10 @@ message EvaluateInstancesResponse {
460468

461469
// Results for trajectory single tool use metric.
462470
TrajectorySingleToolUseResults trajectory_single_tool_use_results = 37;
471+
472+
// Result for rubric based instruction following metric.
473+
RubricBasedInstructionFollowingResult
474+
rubric_based_instruction_following_result = 38;
463475
}
464476
}
465477

@@ -1213,6 +1225,11 @@ message PointwiseMetricInstance {
12131225
// in the json_instance to render
12141226
// PointwiseMetricSpec.instance_prompt_template.
12151227
string json_instance = 1;
1228+
1229+
// Key-value contents for the mutlimodality input, including text, image,
1230+
// video, audio, and pdf, etc. The key is placeholder in metric prompt
1231+
// template, and the value is the multimodal content.
1232+
ContentMap content_map_instance = 2;
12161233
}
12171234
}
12181235

@@ -1225,6 +1242,25 @@ message PointwiseMetricSpec {
12251242
// Optional. System instructions for pointwise metric.
12261243
optional string system_instruction = 2
12271244
[(google.api.field_behavior) = OPTIONAL];
1245+
1246+
// Optional. CustomOutputFormatConfig allows customization of metric output.
1247+
// By default, metrics return a score and explanation.
1248+
// When this config is set, the default output is replaced with either:
1249+
// - The raw output string.
1250+
// - A parsed output based on a user-defined schema.
1251+
// If a custom format is chosen, the `score` and `explanation` fields in the
1252+
// corresponding metric result will be empty.
1253+
CustomOutputFormatConfig custom_output_format_config = 3
1254+
[(google.api.field_behavior) = OPTIONAL];
1255+
}
1256+
1257+
// Spec for custom output format configuration.
1258+
message CustomOutputFormatConfig {
1259+
// Custom output format configuration.
1260+
oneof custom_output_format_config {
1261+
// Optional. Whether to return raw output.
1262+
bool return_raw_output = 1 [(google.api.field_behavior) = OPTIONAL];
1263+
}
12281264
}
12291265

12301266
// Spec for pointwise metric result.
@@ -1234,6 +1270,24 @@ message PointwiseMetricResult {
12341270

12351271
// Output only. Explanation for pointwise metric score.
12361272
string explanation = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
1273+
1274+
// Output only. Spec for custom output.
1275+
CustomOutput custom_output = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
1276+
}
1277+
1278+
// Spec for custom output.
1279+
message CustomOutput {
1280+
// Custom output.
1281+
oneof custom_output {
1282+
// Output only. List of raw output strings.
1283+
RawOutput raw_outputs = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
1284+
}
1285+
}
1286+
1287+
// Raw output.
1288+
message RawOutput {
1289+
// Output only. Raw output string.
1290+
repeated string raw_output = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
12371291
}
12381292

12391293
// Input for pairwise metric.
@@ -1254,6 +1308,11 @@ message PairwiseMetricInstance {
12541308
// in the json_instance to render
12551309
// PairwiseMetricSpec.instance_prompt_template.
12561310
string json_instance = 1;
1311+
1312+
// Key-value contents for the mutlimodality input, including text, image,
1313+
// video, audio, and pdf, etc. The key is placeholder in metric prompt
1314+
// template, and the value is the multimodal content.
1315+
ContentMap content_map_instance = 2;
12571316
}
12581317
}
12591318

@@ -1274,6 +1333,14 @@ message PairwiseMetricSpec {
12741333
// Optional. System instructions for pairwise metric.
12751334
optional string system_instruction = 4
12761335
[(google.api.field_behavior) = OPTIONAL];
1336+
1337+
// Optional. CustomOutputFormatConfig allows customization of metric output.
1338+
// When this config is set, the default output is replaced with
1339+
// the raw output string.
1340+
// If a custom format is chosen, the `pairwise_choice` and `explanation`
1341+
// fields in the corresponding metric result will be empty.
1342+
CustomOutputFormatConfig custom_output_format_config = 5
1343+
[(google.api.field_behavior) = OPTIONAL];
12771344
}
12781345

12791346
// Spec for pairwise metric result.
@@ -1284,6 +1351,9 @@ message PairwiseMetricResult {
12841351

12851352
// Output only. Explanation for pairwise metric score.
12861353
string explanation = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
1354+
1355+
// Output only. Spec for custom output.
1356+
CustomOutput custom_output = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
12871357
}
12881358

12891359
// Input for tool call valid metric.
@@ -1543,6 +1613,53 @@ message MetricxResult {
15431613
optional float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
15441614
}
15451615

1616+
// Instance and metric spec for RubricBasedInstructionFollowing metric.
1617+
message RubricBasedInstructionFollowingInput {
1618+
// Required. Spec for RubricBasedInstructionFollowing metric.
1619+
RubricBasedInstructionFollowingSpec metric_spec = 1
1620+
[(google.api.field_behavior) = REQUIRED];
1621+
1622+
// Required. Instance for RubricBasedInstructionFollowing metric.
1623+
RubricBasedInstructionFollowingInstance instance = 2
1624+
[(google.api.field_behavior) = REQUIRED];
1625+
}
1626+
1627+
// Instance for RubricBasedInstructionFollowing metric - one instance
1628+
// corresponds to one row in an evaluation dataset.
1629+
message RubricBasedInstructionFollowingInstance {
1630+
// Instance for RubricBasedInstructionFollowing metric.
1631+
oneof instance {
1632+
// Required. Instance specified as a json string. String key-value pairs are
1633+
// expected in the json_instance to render RubricBasedInstructionFollowing
1634+
// prompt templates.
1635+
string json_instance = 1 [(google.api.field_behavior) = REQUIRED];
1636+
}
1637+
}
1638+
1639+
// Spec for RubricBasedInstructionFollowing metric - returns rubrics
1640+
// and verdicts corresponding to rubrics along with overall score.
1641+
message RubricBasedInstructionFollowingSpec {}
1642+
1643+
// Result for RubricBasedInstructionFollowing metric.
1644+
message RubricBasedInstructionFollowingResult {
1645+
// Output only. Overall score for the instruction following.
1646+
optional float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
1647+
1648+
// Output only. List of per rubric critique results.
1649+
repeated RubricCritiqueResult rubric_critique_results = 2
1650+
[(google.api.field_behavior) = OUTPUT_ONLY];
1651+
}
1652+
1653+
// Rubric critique result.
1654+
message RubricCritiqueResult {
1655+
// Output only. Rubric to be evaluated.
1656+
string rubric = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
1657+
1658+
// Output only. Verdict for the rubric - true if the rubric is met, false
1659+
// otherwise.
1660+
bool verdict = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
1661+
}
1662+
15461663
// Instances and metric spec for TrajectoryExactMatch metric.
15471664
message TrajectoryExactMatchInput {
15481665
// Required. Spec for TrajectoryExactMatch metric.
@@ -1795,3 +1912,15 @@ message ToolCall {
17951912
// Optional. Spec for tool input
17961913
optional string tool_input = 2 [(google.api.field_behavior) = OPTIONAL];
17971914
}
1915+
1916+
// Map of placeholder in metric prompt template to contents of model input.
1917+
message ContentMap {
1918+
// Repeated Content type.
1919+
message Contents {
1920+
// Optional. Repeated contents.
1921+
repeated Content contents = 1 [(google.api.field_behavior) = OPTIONAL];
1922+
}
1923+
1924+
// Optional. Map of placeholder to contents.
1925+
map<string, Contents> values = 1 [(google.api.field_behavior) = OPTIONAL];
1926+
}

0 commit comments

Comments
 (0)