diff --git a/.github/skills/foundry-extension-scenario-pr-regression/references/impact-mapping.md b/.github/skills/foundry-extension-scenario-pr-regression/references/impact-mapping.md index f8153c842a4..697aea4b9f5 100644 --- a/.github/skills/foundry-extension-scenario-pr-regression/references/impact-mapping.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/references/impact-mapping.md @@ -19,6 +19,7 @@ Files under `internal/cmd/` map to the command they implement: | `internal/cmd/session.go` | `cmd:sessions` | | | `internal/cmd/files.go` | `cmd:files` | | | `internal/cmd/monitor*.go` | `cmd:monitor` | `monitor.go`, `monitor_format.go`. | +| `internal/cmd/insights*.go`, `internal/pkg/agents/insights_api/**` | `cmd:insights` | Tier 0 export help and invalid-severity validation. Live export requires a project with an Insights monitor; no cloud scenario is currently defined. | | `internal/cmd/update.go` | `cmd:endpoint` | `update.go` defines `endpoint update`. | | `internal/cmd/doctor*.go` | `cmd:doctor` | `doctor.go`, `doctor_format.go`. | | `internal/cmd/eval*.go` | `cmd:eval` | `eval.go`, `eval_init.go`, `eval_run.go`, `eval_list.go`, `eval_show.go`, etc. Tier 2 (needs a deployed agent + Foundry endpoint). | diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index b5dd28f7975..c3fc822845c 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -32,6 +32,40 @@ See the shared [AI extension non-interactive input reference](../ai-non-interact for every prompt's flag, environment/configuration input, or deterministic no-prompt behavior. +## Exporting Agent Insights + +Export the current insights for a hosted agent as machine-readable JSON: + +```powershell +$export = azd ai agent insights export | ConvertFrom-Json +$export.insights | Where-Object severity -eq "high" +``` + +The command resolves the agent and `FOUNDRY_PROJECT_ENDPOINT` from the current +azd project and environment. Outside an azd project, pass the Foundry agent name +and project endpoint explicitly: + +```powershell +azd ai agent insights export my-agent ` + --project-endpoint "https://.services.ai.azure.com/api/projects/" ` + --out-file ".\insights.json" + +$export = Get-Content ".\insights.json" -Raw -Encoding utf8 | ConvertFrom-Json +``` + +Use `azd --environment ai agent insights export` to export from a +non-current azd environment. When an agent is resolved from an azd project or +`--environment` is supplied, the endpoint must come from that environment's +`FOUNDRY_PROJECT_ENDPOINT` or an explicit `--project-endpoint`. A missing or +unreadable environment endpoint fails rather than falling back to the global +project context or shell endpoint. + +Exports include highlighted traces, linked traces, and proposed fixes by default. +Use `--include-details=false` for the lightweight insight projection, or filter +the export with `--category`, `--severity`, and `--status`. Treat exported files +as sensitive because insight descriptions and trace details can contain +application or user data. + ## Choosing a Foundry project name During interactive `azd ai agent init`, azd prompts for the name of a new diff --git a/cli/azd/extensions/azure.ai.agents/extension.yaml b/cli/azd/extensions/azure.ai.agents/extension.yaml index f9df1a7cfaf..ef0d0485225 100644 --- a/cli/azd/extensions/azure.ai.agents/extension.yaml +++ b/cli/azd/extensions/azure.ai.agents/extension.yaml @@ -31,3 +31,6 @@ examples: - name: init description: Initialize a new AI agent project. usage: azd ai agent init + - name: export-insights + description: Export an agent's current insights as JSON. + usage: azd ai agent insights export --out-file ./insights.json diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/agent_context.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/agent_context.go index 4f147226cea..3be66b18276 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/agent_context.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/agent_context.go @@ -9,6 +9,7 @@ import ( "fmt" "os" + "azureaiagent/internal/exterrors" "azureaiagent/internal/pkg/agents/agent_api" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -62,6 +63,12 @@ type resolveProjectEndpointOpts struct { // FlagValue is the value of the -p / --project-endpoint flag (level 1). // Empty means the flag was not provided. FlagValue string + // EnvName selects a specific azd environment for level 2. Empty uses the + // current environment. + EnvName string + // RequireEnvironmentEndpoint prevents pairing an environment-bound resource + // with an endpoint from global config or the shell. + RequireEnvironmentEndpoint bool } // resolvedEndpoint holds the result of resolveProjectEndpoint. @@ -73,14 +80,14 @@ type resolvedEndpoint struct { } // azdHostedSources holds the values that the resolver reads from azd-managed -// sources (the active azd environment and ~/.azd/config.json). It is returned +// sources (the selected azd environment and ~/.azd/config.json). It is returned // as a single struct so that tests can stub the whole lookup via // readAzdHostedSourcesFunc. type azdHostedSources struct { - // EnvValue is the FOUNDRY_PROJECT_ENDPOINT value from the active azd - // env, or "" if not set / no active env / no azd client available. + // EnvValue is the FOUNDRY_PROJECT_ENDPOINT value from the selected azd + // env, or "" if not set / no selected env / no azd client available. EnvValue string - // EnvName is the active azd env name. Only meaningful when EnvValue != "". + // EnvName is the selected azd env name. Only meaningful when EnvValue != "". EnvName string // CfgState is the project context persisted in global config. CfgState projectContextState @@ -93,32 +100,49 @@ type azdHostedSources struct { var readAzdHostedSourcesFunc = readAzdHostedSources // readAzdHostedSources dials the azd daemon (if reachable) and reads both -// the active env's FOUNDRY_PROJECT_ENDPOINT and the global-config project -// context in a single client lifetime. Errors talking to the daemon are -// returned only for non-Unavailable cases on the config read — Unavailable -// is treated as "no daemon" and the caller falls through to subsequent levels. -func readAzdHostedSources(ctx context.Context) (azdHostedSources, error) { +// the selected env's FOUNDRY_PROJECT_ENDPOINT and the global-config project +// context in a single client lifetime. Environment-bound resolution fails on +// missing values or read errors instead of consulting unrelated fallback sources. +func readAzdHostedSources(ctx context.Context, opts resolveProjectEndpointOpts) (azdHostedSources, error) { var out azdHostedSources + envName := opts.EnvName + requireEnvironment := opts.RequireEnvironmentEndpoint || envName != "" azdClient, err := azdext.NewAzdClient() if err != nil { + if requireEnvironment { + return out, fmt.Errorf("creating azd client for environment resolution: %w", err) + } // No azd client at all => no hosted sources, not an error. return out, nil } defer azdClient.Close() - if envResp, err := azdClient.Environment().GetCurrent( - ctx, &azdext.EmptyRequest{}, - ); err == nil { + var envResp *azdext.EnvironmentResponse + if envName == "" { + envResp, err = azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + } else { + envResp, err = azdClient.Environment().Get(ctx, &azdext.GetEnvironmentRequest{Name: envName}) + } + if err != nil && requireEnvironment { + return out, fmt.Errorf("getting azd environment %q: %w", envName, err) + } + if err == nil && envResp != nil && envResp.Environment != nil && envResp.Environment.Name != "" { + out.EnvName = envResp.Environment.Name envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ EnvName: envResp.Environment.Name, Key: "FOUNDRY_PROJECT_ENDPOINT", }) - if valErr == nil && envVal.Value != "" { + if valErr != nil && requireEnvironment { + return out, fmt.Errorf("reading FOUNDRY_PROJECT_ENDPOINT from environment %q: %w", out.EnvName, valErr) + } + if valErr == nil && envVal != nil && envVal.Value != "" { out.EnvValue = envVal.Value - out.EnvName = envResp.Environment.Name } } + if requireEnvironment { + return out, nil + } state, found, cfgErr := getProjectContext(ctx, azdClient) if cfgErr != nil { @@ -160,6 +184,8 @@ func containsGRPCCode(err error, code codes.Code) bool { // 5. Structured error with actionable suggestion // // Invalid values at any level produce a hard validation error (no silent fallback). +// An explicit EnvName or RequireEnvironmentEndpoint restricts resolution to the +// selected environment unless FlagValue explicitly overrides it. func resolveProjectEndpoint( ctx context.Context, opts resolveProjectEndpointOpts, @@ -177,7 +203,7 @@ func resolveProjectEndpoint( } // Levels 2 + 3: azd-hosted sources (active env, then global config). - sources, err := readAzdHostedSourcesFunc(ctx) + sources, err := readAzdHostedSourcesFunc(ctx, opts) if err != nil { return nil, err } @@ -194,6 +220,13 @@ func resolveProjectEndpoint( AzdEnvName: sources.EnvName, }, nil } + if opts.RequireEnvironmentEndpoint || opts.EnvName != "" { + return nil, exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "the selected azd environment has no FOUNDRY_PROJECT_ENDPOINT", + "set FOUNDRY_PROJECT_ENDPOINT in that azd environment, or pass --project-endpoint explicitly", + ) + } // Level 3: global config (~/.azd/config.json). if sources.CfgFound && sources.CfgState.Endpoint != "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go index b8794c49ed3..747d3b5d697 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval.go @@ -92,10 +92,10 @@ func addEvalContextFlags(cmd *cobra.Command, flags *evalContextFlags) { "", "Agent service name from azure.yaml, or Foundry agent name outside a project", ) - addEvalProjectEndpointFlag(cmd, &flags.projectEndpoint) + addProjectEndpointFlag(cmd, &flags.projectEndpoint) } -func addEvalProjectEndpointFlag(cmd *cobra.Command, projectEndpoint *string) { +func addProjectEndpointFlag(cmd *cobra.Command, projectEndpoint *string) { cmd.Flags().StringVarP( projectEndpoint, "project-endpoint", diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go index 1f4b7d91ca7..7fc6c4a8176 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_list.go @@ -41,7 +41,7 @@ func newEvalListCommand(extCtx *azdext.ExtensionContext) *cobra.Command { return runEvalList(ctx, flags, extCtx.NoPrompt) }, } - addEvalProjectEndpointFlag(cmd, &flags.projectEndpoint) + addProjectEndpointFlag(cmd, &flags.projectEndpoint) cmd.Flags().IntVar(&flags.limit, "limit", 10, "Maximum number of evals to return") return cmd } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go index f841b7e0245..6937a17ec08 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/eval_show.go @@ -51,7 +51,7 @@ If eval-id is omitted, the most recent eval from the current environment is used return runEvalShow(ctx, evalID, flags, extCtx.NoPrompt) }, } - addEvalProjectEndpointFlag(cmd, &flags.projectEndpoint) + addProjectEndpointFlag(cmd, &flags.projectEndpoint) cmd.Flags().StringVar(&flags.evalRunID, "eval-run-id", "", "Show details for a specific eval run") cmd.Flags().IntVar(&flags.limit, "limit", 20, "Maximum number of runs to show") cmd.Flags().StringVarP(&flags.output, "out-file", "O", "", "Export full run results to a JSON file") diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/helpers.go index 970207bdea6..a2ff5cbcc2e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/helpers.go @@ -997,6 +997,7 @@ type brownfieldAgentExistenceResolver func(context.Context, string, string) (boo type agentServiceResolutionOptions struct { allowBrownfieldInlineName bool brownfieldAgentExists brownfieldAgentExistenceResolver + environmentName string includeProtocolEndpoints bool matchDeployedAgentName bool } @@ -1037,6 +1038,12 @@ func withBrownfieldAgentExistenceResolver( } } +func withEnvironmentName(name string) agentServiceResolutionOption { + return func(options *agentServiceResolutionOptions) { + options.environmentName = name + } +} + // resolveAgentServiceFromProject finds the azure.ai.agent service in azure.yaml // and resolves its deployed agent name and version from the azd environment. // Callers may explicitly opt into the brownfield inline-name fallback; deployed @@ -1072,10 +1079,23 @@ func resolveAgentServiceFromProject( if envValues == nil { // Resolve deployed metadata from azd environment. // Deployed name reflects the created resource. - envResponse, err := azdClient.Environment().GetCurrent( - ctx, &azdext.EmptyRequest{}, - ) + var envResponse *azdext.EnvironmentResponse + if resolutionOptions.environmentName == "" { + envResponse, err = azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + } else { + envResponse, err = azdClient.Environment().Get(ctx, &azdext.GetEnvironmentRequest{ + Name: resolutionOptions.environmentName, + }) + } if err != nil { + if resolutionOptions.environmentName != "" { + return info, fmt.Errorf( + "getting environment %q for agent service %q: %w", + resolutionOptions.environmentName, + svc.Name, + err, + ) + } if resolutionOptions.allowBrownfieldInlineName { return info, fmt.Errorf( "getting current environment for agent service %q: %w", @@ -1087,6 +1107,13 @@ func resolveAgentServiceFromProject( } if envResponse == nil || envResponse.Environment == nil || envResponse.Environment.Name == "" { + if resolutionOptions.environmentName != "" { + return info, fmt.Errorf( + "environment %q is not available for agent service %q", + resolutionOptions.environmentName, + svc.Name, + ) + } if resolutionOptions.allowBrownfieldInlineName { return info, fmt.Errorf( "current environment is not available for agent service %q", diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/insights.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/insights.go new file mode 100644 index 00000000000..cd1e3998489 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/insights.go @@ -0,0 +1,414 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "azureaiagent/internal/exterrors" + "azureaiagent/internal/pkg/agents/insights_api" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +const ( + insightsExportSchemaVersion = "1.0" + insightsPageSize = 100 +) + +type insightsExportFlags struct { + name string + envName string + projectEndpoint string + outFile string + category string + severity string + status string + includeDetails bool +} + +type insightsClient interface { + ListMonitors(context.Context, string, string, int) (*insights_api.MonitorPage, error) + ListInsights( + context.Context, + string, + insights_api.ListInsightOptions, + ) (*insights_api.InsightPage, error) +} + +type insightsExportFilters struct { + Category string `json:"category,omitempty"` + Severity string `json:"severity,omitempty"` + Status string `json:"status,omitempty"` + IncludeDetails bool `json:"include_details"` +} + +type insightsExportDocument struct { + SchemaVersion string `json:"schema_version"` + APIVersion string `json:"api_version"` + ExportedAt string `json:"exported_at"` + ProjectEndpoint string `json:"project_endpoint"` + AgentName string `json:"agent_name"` + MonitorID string `json:"monitor_id"` + Filters insightsExportFilters `json:"filters"` + InsightCount int `json:"insight_count"` + Insights []json.RawMessage `json:"insights"` +} + +type insightsExportAction struct { + client insightsClient + writer io.Writer + now func() time.Time + projectEndpoint string + flags *insightsExportFlags +} + +func newInsightsCommand(extCtx *azdext.ExtensionContext) *cobra.Command { + cmd := &cobra.Command{ + Use: "insights", + Short: "Work with insights generated from hosted-agent traces. (Preview)", + Args: cobra.NoArgs, + } + cmd.AddCommand(newInsightsExportCommand(extCtx)) + return cmd +} + +func newInsightsExportCommand(extCtx *azdext.ExtensionContext) *cobra.Command { + flags := &insightsExportFlags{includeDetails: true} + extCtx = ensureExtensionContext(extCtx) + + cmd := &cobra.Command{ + Use: "export [name]", + Short: "Export insights for a hosted agent as JSON. (Preview)", + Long: `Export all current insights for a hosted agent as JSON. + +The command automatically follows every result page. By default, the export includes +highlighted traces, linked traces, and proposed fixes. Use --include-details=false +to export only the lightweight insight fields. + +When run from an azd project, the agent is resolved from azure.yaml and the active +environment. Optionally specify a service name when the project has multiple agents. +Use --environment to select a different azd environment. Environment-bound exports +require FOUNDRY_PROJECT_ENDPOINT in that environment or an explicit --project-endpoint; +they never fall back to the global project context or shell endpoint. +Outside an azd project, pass the hosted-agent name as the positional argument. + +JSON is written to stdout for PowerShell and other automation. Use --out-file to +write the same document directly to a file. Treat the export as sensitive because +insight descriptions and trace details can contain application or user data.`, + Example: ` # Export the current project's agent insights to stdout + azd ai agent insights export + + # Parse the export in PowerShell + $export = azd ai agent insights export | ConvertFrom-Json + + # Export a named agent, including only active high-severity insights + azd ai agent insights export my-agent --severity high --status active --out-file .\insights.json`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + flags.name = args[0] + } + flags.envName = extCtx.Environment + if err := validateInsightsExportFlags(flags); err != nil { + return err + } + + ctx := azdext.WithAccessToken(cmd.Context()) + azdClient, err := azdext.NewAzdClient() + if err != nil { + return fmt.Errorf("creating azd client: %w", err) + } + defer azdClient.Close() + + info, err := resolveInsightsAgentInfo(ctx, azdClient, flags, extCtx.NoPrompt) + if err != nil { + return err + } + if info.AgentName == "" { + return exterrors.Dependency( + exterrors.CodeAgentDefinitionNotFound, + fmt.Sprintf("agent name could not be resolved for service %q", info.ServiceName), + "run `azd deploy` first, or pass the hosted-agent name explicitly", + ) + } + + resolved, err := resolveInsightsProjectEndpoint(ctx, flags, info) + if err != nil { + return err + } + + credential, err := newAgentCredential() + if err != nil { + return exterrors.Auth( + exterrors.CodeCredentialCreationFailed, + fmt.Sprintf("failed to create an Azure credential: %s", err), + "run `azd auth login` and retry", + ) + } + + action := &insightsExportAction{ + client: insights_api.NewClient(resolved.Endpoint, credential, nil), + writer: cmd.OutOrStdout(), + now: time.Now, + projectEndpoint: resolved.Endpoint, + flags: flags, + } + action.flags.name = info.AgentName + return action.Run(ctx) + }, + } + + addProjectEndpointFlag(cmd, &flags.projectEndpoint) + cmd.Flags().StringVarP(&flags.outFile, "out-file", "O", "", "Write the JSON export to this file") + cmd.Flags().StringVar(&flags.category, "category", "", "Filter by exact insight category") + cmd.Flags().StringVar(&flags.severity, "severity", "", "Filter by severity: high, medium, or low") + cmd.Flags().StringVar(&flags.status, "status", "", "Filter by status: active, resolved, or ignored") + cmd.Flags().BoolVar( + &flags.includeDetails, + "include-details", + true, + "Include traces and proposed fixes in the export", + ) + azdext.RegisterFlagOptions(cmd, azdext.FlagOptions{ + Name: "output", + AllowedValues: []string{"json"}, + Default: "json", + }) + + return cmd +} + +func validateInsightsExportFlags(flags *insightsExportFlags) error { + flags.category = strings.TrimSpace(flags.category) + + var err error + flags.severity, err = normalizeInsightsFilter( + "--severity", + flags.severity, + []string{"high", "medium", "low"}, + ) + if err != nil { + return err + } + flags.status, err = normalizeInsightsFilter( + "--status", + flags.status, + []string{"active", "resolved", "ignored"}, + ) + return err +} + +func resolveInsightsAgentInfo( + ctx context.Context, + azdClient *azdext.AzdClient, + flags *insightsExportFlags, + noPrompt bool, +) (*AgentServiceInfo, error) { + return resolveMonitorAgentInfo( + ctx, + azdClient, + flags.name, + noPrompt, + withEnvironmentName(flags.envName), + ) +} + +func resolveInsightsProjectEndpoint( + ctx context.Context, + flags *insightsExportFlags, + info *AgentServiceInfo, +) (*resolvedEndpoint, error) { + environmentBound := info.ServiceName != "" || flags.envName != "" + resolved, err := resolveProjectEndpoint(ctx, resolveProjectEndpointOpts{ + FlagValue: flags.projectEndpoint, + EnvName: flags.envName, + RequireEnvironmentEndpoint: environmentBound, + }) + if err != nil && !environmentBound { + return nil, insightsEndpointError(err) + } + return resolved, err +} + +func normalizeInsightsFilter(flagName, value string, allowed []string) (string, error) { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + return "", nil + } + if slices.Contains(allowed, value) { + return value, nil + } + return "", exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("%s must be one of: %s", flagName, strings.Join(allowed, ", ")), + fmt.Sprintf("choose a supported value for %s", flagName), + ) +} + +func (a *insightsExportAction) Run(ctx context.Context) error { + monitorPage, err := a.client.ListMonitors(ctx, a.flags.name, "", 2) + if err != nil { + return exterrors.ServiceFromAzure(err, exterrors.OpListInsightMonitors) + } + matches := make([]insights_api.Monitor, 0, len(monitorPage.Data)) + for _, monitor := range monitorPage.Data { + if strings.EqualFold(monitor.AgentName, a.flags.name) { + matches = append(matches, monitor) + } + } + if len(matches) == 0 { + return exterrors.Dependency( + exterrors.CodeInsightMonitorNotFound, + fmt.Sprintf("no Agent Insights monitor exists for agent %q", a.flags.name), + "create an Insights monitor for this agent in Microsoft Foundry, then retry", + ) + } + if len(matches) > 1 || monitorPage.HasMore { + return exterrors.Dependency( + exterrors.CodeInsightMonitorAmbiguous, + fmt.Sprintf("multiple Agent Insights monitors were returned for agent %q", a.flags.name), + "remove duplicate monitors for this agent in Microsoft Foundry, then retry", + ) + } + + monitor := matches[0] + if monitor.ID == "" { + return exterrors.Internal( + exterrors.CodeInvalidInsightMonitor, + fmt.Sprintf("Agent Insights returned a monitor without an ID for agent %q", monitor.AgentName), + ) + } + + monitorID := monitor.ID + insights, err := fetchAllInsights(ctx, a.client, monitorID, a.flags) + if err != nil { + return exterrors.ServiceFromAzure(err, exterrors.OpListInsights) + } + + document := insightsExportDocument{ + SchemaVersion: insightsExportSchemaVersion, + APIVersion: insights_api.APIVersion, + ExportedAt: a.now().UTC().Format(time.RFC3339), + ProjectEndpoint: a.projectEndpoint, + AgentName: monitor.AgentName, + MonitorID: monitorID, + Filters: insightsExportFilters{ + Category: a.flags.category, + Severity: a.flags.severity, + Status: a.flags.status, + IncludeDetails: a.flags.includeDetails, + }, + InsightCount: len(insights), + Insights: insights, + } + + if a.flags.outFile != "" { + if err := writeInsightsExport(a.flags.outFile, document); err != nil { + return fmt.Errorf("writing insights export: %w", err) + } + return nil + } + + encoder := json.NewEncoder(a.writer) + encoder.SetIndent("", " ") + if err := encoder.Encode(document); err != nil { + return fmt.Errorf("writing insights export: %w", err) + } + return nil +} + +func fetchAllInsights( + ctx context.Context, + client insightsClient, + monitorID string, + flags *insightsExportFlags, +) ([]json.RawMessage, error) { + insights := make([]json.RawMessage, 0) + after := "" + seenCursors := map[string]struct{}{} + + for { + page, err := client.ListInsights(ctx, monitorID, insights_api.ListInsightOptions{ + Category: flags.category, + Severity: flags.severity, + Status: flags.status, + IncludeDetails: flags.includeDetails, + Order: "desc", + After: after, + Limit: insightsPageSize, + }) + if err != nil { + return nil, err + } + insights = append(insights, page.Data...) + if !page.HasMore { + return insights, nil + } + + next := strings.TrimSpace(page.LastID) + if next == "" { + return nil, fmt.Errorf("Agent Insights returned has_more without a continuation cursor") + } + if _, exists := seenCursors[next]; exists { + return nil, fmt.Errorf("Agent Insights returned a repeated continuation cursor %q", next) + } + seenCursors[next] = struct{}{} + after = next + } +} + +func writeInsightsExport(path string, document insightsExportDocument) (err error) { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating output directory: %w", err) + } + + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0o600) + if err != nil { + return fmt.Errorf("opening output file: %w", err) + } + defer func() { + err = errors.Join(err, file.Close()) + }() + if err := file.Chmod(0o600); err != nil { + return fmt.Errorf("restricting output file permissions: %w", err) + } + if err := file.Truncate(0); err != nil { + return fmt.Errorf("truncating output file: %w", err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("seeking output file: %w", err) + } + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + if err := encoder.Encode(document); err != nil { + return fmt.Errorf("writing output file: %w", err) + } + return nil +} + +func insightsEndpointError(err error) error { + localErr, ok := errors.AsType[*azdext.LocalError](err) + if !ok || localErr.Code != exterrors.CodeMissingProjectEndpoint { + return err + } + + return exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "no Foundry project endpoint is configured", + "pass --project-endpoint, set a default with `azd ai project set `, "+ + "or run from an azd project with FOUNDRY_PROJECT_ENDPOINT set", + ) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/insights_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/insights_test.go new file mode 100644 index 00000000000..e2f484a913e --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/insights_test.go @@ -0,0 +1,390 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaiagent/internal/exterrors" + "azureaiagent/internal/pkg/agents/insights_api" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type fakeInsightsClient struct { + monitorPage *insights_api.MonitorPage + monitorErr error + pages map[string]*insights_api.InsightPage + pageErr error + options []insights_api.ListInsightOptions +} + +func (f *fakeInsightsClient) ListMonitors( + context.Context, + string, + string, + int, +) (*insights_api.MonitorPage, error) { + return f.monitorPage, f.monitorErr +} + +func (f *fakeInsightsClient) ListInsights( + _ context.Context, + _ string, + options insights_api.ListInsightOptions, +) (*insights_api.InsightPage, error) { + f.options = append(f.options, options) + if f.pageErr != nil { + return nil, f.pageErr + } + page, ok := f.pages[options.After] + if !ok { + return nil, fmt.Errorf("unexpected cursor %q", options.After) + } + return page, nil +} + +func TestInsightsExportCommandArguments(t *testing.T) { + cmd := newInsightsExportCommand(nil) + + assert.NoError(t, cmd.Args(cmd, nil)) + assert.NoError(t, cmd.Args(cmd, []string{"agent"})) + assert.Error(t, cmd.Args(cmd, []string{"agent", "extra"})) + assertOutputFlagOptions(t, cmd, "json", []string{"json"}) +} + +func TestInsightsCommandIncludesExport(t *testing.T) { + cmd := newInsightsCommand(nil) + + export, _, err := cmd.Find([]string{"export"}) + + require.NoError(t, err) + assert.Equal(t, "export [name]", export.Use) +} + +func TestValidateInsightsExportFlagsNormalizesEnums(t *testing.T) { + flags := &insightsExportFlags{ + category: " aviation ", + severity: " HIGH ", + status: "Resolved", + } + + err := validateInsightsExportFlags(flags) + + require.NoError(t, err) + assert.Equal(t, "aviation", flags.category) + assert.Equal(t, "high", flags.severity) + assert.Equal(t, "resolved", flags.status) +} + +func TestValidateInsightsExportFlagsRejectsUnsupportedValue(t *testing.T) { + flags := &insightsExportFlags{severity: "critical"} + + err := validateInsightsExportFlags(flags) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, "invalid_parameter", localErr.Code) +} + +func TestResolveInsightsAgentInfoUsesExplicitAgentOutsideProject(t *testing.T) { + azdClient := newHelpersTestAzdClient( + t, + &helpersProjectServer{err: status.Error(codes.NotFound, "project not found")}, + &helpersPromptServer{}, + ) + flags := &insightsExportFlags{ + name: "remote-agent", + projectEndpoint: "https://account.services.ai.azure.com/api/projects/project", + } + + info, err := resolveInsightsAgentInfo(t.Context(), azdClient, flags, true) + + require.NoError(t, err) + assert.Equal(t, "remote-agent", info.AgentName) +} + +func TestResolveInsightsAgentInfoUsesServiceFromRequestedEnvironment(t *testing.T) { + projectServer := &helpersProjectServer{project: &azdext.ProjectConfig{ + Path: t.TempDir(), + Services: map[string]*azdext.ServiceConfig{ + "service-key": { + Name: "service-key", + Host: AiAgentHost, + }, + }, + }} + envServer := &testEnvironmentServiceServer{ + current: &azdext.Environment{Name: "dev"}, + environments: map[string]*azdext.Environment{ + "staging": {Name: "staging"}, + }, + values: map[string]map[string]string{ + "dev": { + "AGENT_SERVICE_KEY_NAME": "dev-agent", + }, + "staging": { + "AGENT_SERVICE_KEY_NAME": "staging-agent", + }, + }, + } + azdClient := newHelpersTestAzdClient( + t, + projectServer, + &helpersPromptServer{}, + envServer, + ) + flags := &insightsExportFlags{ + name: "service-key", + envName: "staging", + projectEndpoint: "https://account.services.ai.azure.com/api/projects/project", + } + + info, err := resolveInsightsAgentInfo(t.Context(), azdClient, flags, true) + + require.NoError(t, err) + assert.Equal(t, "staging-agent", info.AgentName) +} + +func TestResolveInsightsProjectEndpointBindsEnvironment(t *testing.T) { + for _, tc := range []struct { + name string + envName string + serviceName string + }{ + {name: "explicit environment", envName: "staging"}, + {name: "current project environment", serviceName: "service-key"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://shell.services.ai.azure.com/api/projects/other") + original := readAzdHostedSourcesFunc + readAzdHostedSourcesFunc = func( + _ context.Context, opts resolveProjectEndpointOpts, + ) (azdHostedSources, error) { + assert.True(t, opts.RequireEnvironmentEndpoint) + assert.Equal(t, tc.envName, opts.EnvName) + return azdHostedSources{ + CfgFound: true, + CfgState: projectContextState{Endpoint: "https://global.services.ai.azure.com/api/projects/other"}, + }, nil + } + t.Cleanup(func() { readAzdHostedSourcesFunc = original }) + + resolved, err := resolveInsightsProjectEndpoint(t.Context(), + &insightsExportFlags{envName: tc.envName}, &AgentServiceInfo{ServiceName: tc.serviceName}) + + require.Nil(t, resolved) + localErr, ok := errors.AsType[*azdext.LocalError](err) + require.True(t, ok) + assert.Equal(t, exterrors.CodeMissingProjectEndpoint, localErr.Code) + assert.Contains(t, localErr.Suggestion, "--project-endpoint") + assert.NotContains(t, localErr.Suggestion, "azd ai project set") + }) + } +} + +func TestResolveInsightsProjectEndpointExplicitOverride(t *testing.T) { + stubAzdHostedSources(t, azdHostedSources{}, errors.New("environment must not be read")) + endpoint := "https://explicit.services.ai.azure.com/api/projects/project" + + resolved, err := resolveInsightsProjectEndpoint(t.Context(), + &insightsExportFlags{envName: "staging", projectEndpoint: endpoint}, + &AgentServiceInfo{ServiceName: "service-key"}) + + require.NoError(t, err) + assert.Equal(t, endpoint, resolved.Endpoint) + assert.Equal(t, SourceFlag, resolved.Source) +} + +func TestResolveInsightsProjectEndpointStandaloneFallback(t *testing.T) { + stubAzdHostedSources(t, azdHostedSources{ + CfgFound: true, + CfgState: projectContextState{Endpoint: "https://global.services.ai.azure.com/api/projects/project"}, + }, nil) + + resolved, err := resolveInsightsProjectEndpoint(t.Context(), + &insightsExportFlags{}, &AgentServiceInfo{AgentName: "remote-agent"}) + + require.NoError(t, err) + assert.Equal(t, SourceGlobalConfig, resolved.Source) +} + +func TestInsightsExportActionWritesAllPagesToStdout(t *testing.T) { + client := &fakeInsightsClient{ + monitorPage: &insights_api.MonitorPage{ + Data: []insights_api.Monitor{{ID: "monitor-1", AgentName: "Agent"}}, + }, + pages: map[string]*insights_api.InsightPage{ + "": { + Data: []json.RawMessage{json.RawMessage(`{"id":"insight-1","extra":"kept"}`)}, + LastID: "insight-1", + HasMore: true, + }, + "insight-1": { + Data: []json.RawMessage{json.RawMessage(`{"id":"insight-2"}`)}, + }, + }, + } + var output bytes.Buffer + action := &insightsExportAction{ + client: client, + writer: &output, + now: func() time.Time { return time.Date(2026, 9, 9, 20, 0, 0, 0, time.UTC) }, + projectEndpoint: "https://account.services.ai.azure.com/api/projects/project", + flags: &insightsExportFlags{ + name: "agent", + category: "aviation", + severity: "high", + status: "active", + includeDetails: true, + }, + } + + err := action.Run(t.Context()) + + require.NoError(t, err) + var document insightsExportDocument + require.NoError(t, json.Unmarshal(output.Bytes(), &document)) + assert.Equal(t, "v1", document.APIVersion) + assert.Equal(t, "2026-09-09T20:00:00Z", document.ExportedAt) + assert.Equal(t, "Agent", document.AgentName) + assert.Equal(t, "monitor-1", document.MonitorID) + assert.Equal(t, 2, document.InsightCount) + require.Len(t, document.Insights, 2) + assert.JSONEq(t, `{"id":"insight-1","extra":"kept"}`, string(document.Insights[0])) + require.Len(t, client.options, 2) + assert.Empty(t, client.options[0].After) + assert.Equal(t, "insight-1", client.options[1].After) + assert.True(t, client.options[0].IncludeDetails) +} + +func TestInsightsExportActionWritesFile(t *testing.T) { + outFile := filepath.Join(t.TempDir(), "nested", "insights.json") + client := &fakeInsightsClient{ + monitorPage: &insights_api.MonitorPage{ + Data: []insights_api.Monitor{{ID: "monitor-1", AgentName: "agent"}}, + }, + pages: map[string]*insights_api.InsightPage{"": {}}, + } + action := &insightsExportAction{ + client: client, + writer: &bytes.Buffer{}, + now: func() time.Time { return time.Unix(0, 0) }, + projectEndpoint: "https://account.services.ai.azure.com/api/projects/project", + flags: &insightsExportFlags{ + name: "agent", + outFile: outFile, + includeDetails: true, + }, + } + + err := action.Run(t.Context()) + + require.NoError(t, err) + content, err := os.ReadFile(outFile) + require.NoError(t, err) + assert.Contains(t, string(content), `"insight_count": 0`) + assert.Contains(t, string(content), `"insights": []`) +} + +func TestInsightsExportActionRequiresMonitor(t *testing.T) { + action := &insightsExportAction{ + client: &fakeInsightsClient{ + monitorPage: &insights_api.MonitorPage{}, + }, + writer: &bytes.Buffer{}, + now: time.Now, + flags: &insightsExportFlags{name: "agent"}, + } + + err := action.Run(t.Context()) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, "insight_monitor_not_found", localErr.Code) +} + +func TestInsightsExportActionRejectsMismatchedMonitor(t *testing.T) { + action := &insightsExportAction{ + client: &fakeInsightsClient{ + monitorPage: &insights_api.MonitorPage{ + Data: []insights_api.Monitor{{ID: "monitor-other", AgentName: "other-agent"}}, + }, + }, + writer: &bytes.Buffer{}, + now: time.Now, + flags: &insightsExportFlags{name: "requested-agent"}, + } + + err := action.Run(t.Context()) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, "insight_monitor_not_found", localErr.Code) +} + +func TestInsightsExportActionRejectsAmbiguousMonitor(t *testing.T) { + action := &insightsExportAction{ + client: &fakeInsightsClient{ + monitorPage: &insights_api.MonitorPage{ + Data: []insights_api.Monitor{ + {ID: "monitor-1", AgentName: "agent"}, + {ID: "monitor-2", AgentName: "agent"}, + }, + }, + }, + writer: &bytes.Buffer{}, + now: time.Now, + flags: &insightsExportFlags{name: "agent"}, + } + + err := action.Run(t.Context()) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, "insight_monitor_ambiguous", localErr.Code) +} + +func TestFetchAllInsightsRejectsMissingContinuationCursor(t *testing.T) { + client := &fakeInsightsClient{ + pages: map[string]*insights_api.InsightPage{ + "": {HasMore: true}, + }, + } + + _, err := fetchAllInsights(t.Context(), client, "monitor-1", &insightsExportFlags{}) + + require.ErrorContains(t, err, "without a continuation cursor") +} + +func TestFetchAllInsightsRejectsRepeatedContinuationCursor(t *testing.T) { + client := &fakeInsightsClient{ + pages: map[string]*insights_api.InsightPage{ + "": { + LastID: "insight-1", + HasMore: true, + }, + "insight-1": { + LastID: "insight-1", + HasMore: true, + }, + }, + } + + _, err := fetchAllInsights(t.Context(), client, "monitor-1", &insightsExportFlags{}) + + require.ErrorContains(t, err, "repeated continuation cursor") +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/monitor.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/monitor.go index a256577a70f..7c1515ff282 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/monitor.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/monitor.go @@ -157,8 +157,9 @@ func resolveMonitorAgentInfo( azdClient *azdext.AzdClient, name string, noPrompt bool, + options ...agentServiceResolutionOption, ) (*AgentServiceInfo, error) { - info, err := resolveAgentServiceFromProject(ctx, azdClient, name, noPrompt) + info, err := resolveAgentServiceFromProject(ctx, azdClient, name, noPrompt, options...) if err == nil { return info, nil } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/project_resolver_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/project_resolver_test.go index aa0bce97bfe..e72542bd1f2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/project_resolver_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/project_resolver_test.go @@ -13,6 +13,8 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // stubAzdHostedSources replaces readAzdHostedSourcesFunc for the duration of @@ -20,7 +22,7 @@ import ( func stubAzdHostedSources(t *testing.T, sources azdHostedSources, err error) { t.Helper() orig := readAzdHostedSourcesFunc - readAzdHostedSourcesFunc = func(context.Context) (azdHostedSources, error) { + readAzdHostedSourcesFunc = func(context.Context, resolveProjectEndpointOpts) (azdHostedSources, error) { return sources, err } t.Cleanup(func() { readAzdHostedSourcesFunc = orig }) @@ -77,6 +79,83 @@ func TestResolveProjectEndpoint_AzdEnvResolves(t *testing.T) { assert.Equal(t, "dev", result.AzdEnvName) } +func TestResolveProjectEndpoint_UsesRequestedAzdEnvironment(t *testing.T) { + var requestedEnv string + orig := readAzdHostedSourcesFunc + readAzdHostedSourcesFunc = func(_ context.Context, opts resolveProjectEndpointOpts) (azdHostedSources, error) { + requestedEnv = opts.EnvName + return azdHostedSources{ + EnvValue: "https://staging.services.ai.azure.com/api/projects/p", + EnvName: opts.EnvName, + }, nil + } + t.Cleanup(func() { readAzdHostedSourcesFunc = orig }) + + result, err := resolveProjectEndpoint(t.Context(), resolveProjectEndpointOpts{EnvName: "staging"}) + + require.NoError(t, err) + assert.Equal(t, "staging", requestedEnv) + assert.Equal(t, "https://staging.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, "staging", result.AzdEnvName) +} + +func TestResolveProjectEndpoint_EnvironmentBoundLookup(t *testing.T) { + for _, tc := range []struct { + name string + envName string + endpoint string + readError bool + wantError string + }{ + {name: "selected", envName: "staging", endpoint: "https://staging.services.ai.azure.com/api/projects/p"}, + {name: "current", endpoint: "https://dev.services.ai.azure.com/api/projects/p"}, + {name: "missing selected value", envName: "staging", wantError: "no FOUNDRY_PROJECT_ENDPOINT"}, + {name: "missing current value", wantError: "no FOUNDRY_PROJECT_ENDPOINT"}, + {name: "missing environment", envName: "missing", wantError: "getting azd environment"}, + {name: "value read failure", envName: "staging", readError: true, wantError: "reading FOUNDRY_PROJECT_ENDPOINT"}, + {name: "invalid value", envName: "staging", endpoint: "http://invalid", wantError: "https"}, + } { + t.Run(tc.name, func(t *testing.T) { + selected := tc.envName + if selected == "" { + selected = "dev" + } + env := testEnvironmentServiceServer{ + current: &azdext.Environment{Name: "dev"}, + environments: map[string]*azdext.Environment{ + "staging": {Name: "staging"}, + }, + values: map[string]map[string]string{ + selected: {"FOUNDRY_PROJECT_ENDPOINT": tc.endpoint}, + }, + } + var envServer azdext.EnvironmentServiceServer = &env + if tc.readError { + envServer = &helpersFailingEnvironmentServer{ + testEnvironmentServiceServer: env, + getValueErr: status.Error(codes.Unavailable, "read failed"), + } + } + address := newTestAzdServer(t, envServer, &testWorkflowServiceServer{}) + t.Setenv("AZD_SERVER", address) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://shell.services.ai.azure.com/api/projects/other") + + resolved, err := resolveProjectEndpoint(t.Context(), resolveProjectEndpointOpts{ + EnvName: tc.envName, RequireEnvironmentEndpoint: true, + }) + + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + require.Nil(t, resolved) + } else { + require.NoError(t, err) + assert.Equal(t, tc.endpoint, resolved.Endpoint) + assert.Equal(t, selected, resolved.AzdEnvName) + } + }) + } +} + func TestResolveProjectEndpoint_AzdEnvInvalidRejected(t *testing.T) { // Level 2 invalid values are hard errors (no silent fallback to lower levels). // FOUNDRY_PROJECT_ENDPOINT is set, but resolver must NOT fall through to it. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/root.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/root.go index 0dd0d2b23f7..7bf85220583 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/root.go @@ -68,6 +68,7 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand(newSessionCommand(extCtx)) rootCmd.AddCommand(newSampleCommand(extCtx)) rootCmd.AddCommand(newDoctorCommand()) + rootCmd.AddCommand(newInsightsCommand(extCtx)) rootCmd.AddCommand(newCodeCommand(extCtx)) rootCmd.AddCommand(newEvalCommand(extCtx)) diff --git a/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go index a77308df35a..aed8a9065f9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go +++ b/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go @@ -67,6 +67,8 @@ const ( CodeMissingAgentEnvVars = "missing_agent_env_vars" CodeFoundryDependencyNotReady = "foundry_dependency_not_ready" CodeMissingProjectEndpoint = "missing_project_endpoint" + CodeInsightMonitorNotFound = "insight_monitor_not_found" + CodeInsightMonitorAmbiguous = "insight_monitor_ambiguous" CodeGitHubDownloadFailed = "github_download_failed" CodePromptFailed = "prompt_failed" ) @@ -192,6 +194,7 @@ const ( CodeAgentCreateFailed = "agent_create_failed" CodeMsaAppIDAlreadyInUse = "msa_app_id_already_in_use" CodeMultipleBotsForMsaAppID = "multiple_bots_for_msa_app_id" + CodeInvalidInsightMonitor = "invalid_insight_monitor" ) // Operation names for [ServiceFromAzure] errors. @@ -217,6 +220,8 @@ const ( OpDeleteSession = "delete_session" OpStopSession = "stop_session" OpListSessions = "list_sessions" + OpListInsightMonitors = "list_insight_monitors" + OpListInsights = "list_insights" OpCreateToolboxVersion = "create_toolbox_version" OpGetToolbox = "get_toolbox" OpProvisionMemoryStore = "provision_memory_store" diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/client.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/client.go new file mode 100644 index 00000000000..abb9e79c471 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/client.go @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package insights_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" + + "azureaiagent/internal/pkg/useragent" +) + +const ( + monitorPath = "/agent_insight_monitors" + insightsFeatureHeader = "AgentInsights=V1Preview" +) + +type foundryFeaturesPolicy struct{} + +func (foundryFeaturesPolicy) Do(req *policy.Request) (*http.Response, error) { + req.Raw().Header.Set("Foundry-Features", insightsFeatureHeader) + return req.Next() +} + +// Client accesses Agent Insights through a Microsoft Foundry project endpoint. +type Client struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewClient creates an authenticated Agent Insights client. +// A nil transport uses the Azure SDK's default HTTP transport. +func NewClient(endpoint string, credential azcore.TokenCredential, transport policy.Transporter) *Client { + clientOptions := &policy.ClientOptions{ + Transport: transport, + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(credential, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(useragent.Default()), + foundryFeaturesPolicy{}, + }, + } + + return &Client{ + endpoint: endpoint, + pipeline: runtime.NewPipeline( + "azure-ai-agent-insights", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ), + } +} + +// ListMonitors returns monitors matching an exact agent name. +func (c *Client) ListMonitors( + ctx context.Context, + agentName string, + after string, + limit int, +) (*MonitorPage, error) { + query := url.Values{} + query.Set("api-version", APIVersion) + query.Set("agent_name", agentName) + query.Set("order", "desc") + if after != "" { + query.Set("after", after) + } + if limit > 0 { + query.Set("limit", strconv.Itoa(limit)) + } + + var page MonitorPage + if err := c.get(ctx, monitorPath, query, &page); err != nil { + return nil, err + } + return &page, nil +} + +// ListInsights returns one page of insights for a monitor. +func (c *Client) ListInsights( + ctx context.Context, + monitorID string, + options ListInsightOptions, +) (*InsightPage, error) { + query := url.Values{} + query.Set("api-version", APIVersion) + query.Set("include_details", strconv.FormatBool(options.IncludeDetails)) + if options.Category != "" { + query.Set("category", options.Category) + } + if options.Severity != "" { + query.Set("severity", options.Severity) + } + if options.Status != "" { + query.Set("status", options.Status) + } + if options.Order != "" { + query.Set("order", options.Order) + } + if options.After != "" { + query.Set("after", options.After) + } + if options.Limit > 0 { + query.Set("limit", strconv.Itoa(options.Limit)) + } + + path := monitorPath + "/" + url.PathEscape(monitorID) + "/insights" + var page InsightPage + if err := c.get(ctx, path, query, &page); err != nil { + return nil, err + } + return &page, nil +} + +func (c *Client) get(ctx context.Context, path string, query url.Values, result any) error { + endpoint, err := url.Parse(c.endpoint) + if err != nil { + return fmt.Errorf("invalid project endpoint: %w", err) + } + if endpoint.Scheme == "" || endpoint.Host == "" { + return fmt.Errorf("invalid project endpoint: scheme and host are required") + } + requestURL := endpoint.Scheme + "://" + endpoint.Host + + strings.TrimRight(endpoint.EscapedPath(), "/") + path + if encodedQuery := query.Encode(); encodedQuery != "" { + requestURL += "?" + encodedQuery + } + + req, err := runtime.NewRequest(ctx, http.MethodGet, requestURL) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return fmt.Errorf("sending request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if !runtime.HasStatusCode(resp, http.StatusOK) { + resp.Body = io.NopCloser(bytes.NewReader(body)) + return runtime.NewResponseError(resp) + } + if err := json.Unmarshal(body, result); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/client_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/client_test.go new file mode 100644 index 00000000000..6c615233dc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/client_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package insights_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeCredential struct{} + +func (fakeCredential) GetToken( + context.Context, + policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "fake-token", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +func newTestClient(t *testing.T, handler http.Handler) *Client { + t.Helper() + server := httptest.NewTLSServer(handler) + t.Cleanup(server.Close) + return NewClient(server.URL+"/api/projects/project", fakeCredential{}, server.Client()) +} + +func TestNewClient(t *testing.T) { + t.Parallel() + + client := NewClient("https://example.services.ai.azure.com/api/projects/project", fakeCredential{}, nil) + + require.NotNil(t, client) + assert.Equal(t, "https://example.services.ai.azure.com/api/projects/project", client.endpoint) +} + +func TestListMonitors(t *testing.T) { + t.Parallel() + + client := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/projects/project/agent_insight_monitors", r.URL.Path) + assert.Equal(t, APIVersion, r.URL.Query().Get("api-version")) + assert.Equal(t, "travel agent", r.URL.Query().Get("agent_name")) + assert.Equal(t, "cursor-1", r.URL.Query().Get("after")) + assert.Equal(t, "2", r.URL.Query().Get("limit")) + assert.Equal(t, "desc", r.URL.Query().Get("order")) + assert.Equal(t, "AgentInsights=V1Preview", r.Header.Get("Foundry-Features")) + assert.Equal(t, "Bearer fake-token", r.Header.Get("Authorization")) + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{{ + "id": "monitor-1", + "agent_name": "travel agent", + }}, + "last_id": "monitor-1", + "has_more": false, + }) + })) + + page, err := client.ListMonitors(t.Context(), "travel agent", "cursor-1", 2) + + require.NoError(t, err) + require.Len(t, page.Data, 1) + assert.Equal(t, "monitor-1", page.Data[0].ID) +} + +func TestListInsightsPreservesUnknownFields(t *testing.T) { + t.Parallel() + + client := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/projects/project/agent_insight_monitors/monitor-1/insights", r.URL.Path) + assert.Equal(t, APIVersion, r.URL.Query().Get("api-version")) + assert.Equal(t, "aviation", r.URL.Query().Get("category")) + assert.Equal(t, "high", r.URL.Query().Get("severity")) + assert.Equal(t, "active", r.URL.Query().Get("status")) + assert.Equal(t, "true", r.URL.Query().Get("include_details")) + assert.Equal(t, "desc", r.URL.Query().Get("order")) + assert.Equal(t, "cursor-1", r.URL.Query().Get("after")) + assert.Equal(t, "100", r.URL.Query().Get("limit")) + assert.Equal(t, "AgentInsights=V1Preview", r.Header.Get("Foundry-Features")) + assert.Equal(t, "Bearer fake-token", r.Header.Get("Authorization")) + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "data": [{"id":"insight-1","future_field":{"preserved":true}}], + "last_id":"insight-1", + "has_more":false + }`)) + })) + + page, err := client.ListInsights(t.Context(), "monitor-1", ListInsightOptions{ + Category: "aviation", + Severity: "high", + Status: "active", + IncludeDetails: true, + Order: "desc", + After: "cursor-1", + Limit: 100, + }) + + require.NoError(t, err) + require.Len(t, page.Data, 1) + assert.JSONEq(t, `{"id":"insight-1","future_field":{"preserved":true}}`, string(page.Data[0])) +} + +func TestListInsightsReturnsResponseError(t *testing.T) { + t.Parallel() + + client := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":{"code":"Forbidden","message":"denied"}}`)) + })) + + _, err := client.ListInsights(t.Context(), "monitor-1", ListInsightOptions{}) + + var responseError *azcore.ResponseError + require.ErrorAs(t, err, &responseError) + assert.Equal(t, http.StatusForbidden, responseError.StatusCode) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/models.go new file mode 100644 index 00000000000..d29fe156781 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/insights_api/models.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package insights_api contains the Agent Insights data-plane client and contracts. +package insights_api + +import "encoding/json" + +// APIVersion is the Agent Insights public data-plane API version. +const APIVersion = "v1" + +// Monitor is the subset of an Agent Insights monitor needed to locate an agent's insights. +type Monitor struct { + ID string `json:"id"` + AgentName string `json:"agent_name"` +} + +// MonitorPage is a cursor-paginated monitor response. +type MonitorPage struct { + Data []Monitor `json:"data"` + LastID string `json:"last_id"` + HasMore bool `json:"has_more"` +} + +// InsightPage is a cursor-paginated insight response. +// +// Insights remain raw JSON so exports preserve fields introduced by newer service versions. +type InsightPage struct { + Data []json.RawMessage `json:"data"` + LastID string `json:"last_id"` + HasMore bool `json:"has_more"` +} + +// ListInsightOptions controls server-side filtering and cursor pagination. +type ListInsightOptions struct { + Category string + Severity string + Status string + IncludeDetails bool + Order string + After string + Limit int +} diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 129e488f3ff..7c643dcef41 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -421,6 +421,11 @@ Invocations-capable setup and lifecycle. ## Tags +`cmd:insights` selects the offline export-help and invalid-severity scenarios +(`tier0/0.17-insights-export-help.yaml` and +`tier0/0.18-insights-export-validate-severity.yaml`). Live export coverage requires +an existing Insights monitor and is not part of the shared-agent setup. + Every scenario carries a top-level `tags:` list so an orchestrator can pick subsets via the tester's `list_scenarios` MCP tool. The tool's filter is **OR across the requested tags, case-sensitive, exact match**: a scenario matches diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.17-insights-export-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.17-insights-export-help.yaml new file mode 100644 index 00000000000..cb47516f687 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.17-insights-export-help.yaml @@ -0,0 +1,14 @@ +# Tier 0 (offline) - verify Insights export command discovery and documented flags. +name: "insights-export-help" +command: "azd ai agent insights export --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:insights", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output describes exporting insights as JSON and shows export [name] in the usage line." + - "Confirm the output lists --out-file, --project-endpoint, --category, --severity, --status, and --include-details." + - "Confirm the help explains environment-bound endpoint resolution and includes a PowerShell ConvertFrom-Json example." + - "Confirm the process exits with code 0 without prompting for authentication, selecting a project, or producing a stack trace." + - "Take a screenshot of the help output." + - "Report a finding if the command is unknown, expected help is missing, a product prompt appears, or the exit code is non-zero." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.18-insights-export-validate-severity.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.18-insights-export-validate-severity.yaml new file mode 100644 index 00000000000..a784ad2d182 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.18-insights-export-validate-severity.yaml @@ -0,0 +1,13 @@ +# Tier 0 (offline) - reject invalid severity before authentication or project resolution. +name: "insights-export-validate-severity" +command: "azd ai agent insights export --severity critical" +cwd: "/tmp" +tags: ["tier:0", "cmd:insights", "negative-path", "parallel-safe"] + +goals: + - "Wait for the command to reject the unsupported severity value." + - "Confirm it exits non-zero with an error naming --severity and listing high, medium, and low as the supported values." + - "Confirm no product prompt appears and the error is severity validation, not missing authentication, project configuration, or an Insights monitor." + - "Confirm no JSON export is emitted and no stack trace appears." + - "Take a screenshot of the error output." + - "Report a finding if invalid severity is accepted, a product prompt appears, or resolution or authentication fails before severity validation."