Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cli/testdata/server-config.yaml.golden
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,10 @@ chat:
# opt-in settings.
# (default: false, type: bool)
debugLoggingEnabled: false
# How long a chat model attempt may go without receiving a stream part before it
# is canceled and retried.
# (default: 5m, type: duration)
streamSilenceTimeout: 5m0s
# HTTPS URL to receive chat agent lifecycle hook events (plain HTTP requires
# --chat-hook-allow-insecure). Hooks are disabled when unset. Requires the
# agent-lifecycle-hooks experiment.
Expand Down
3 changes: 3 additions & 0 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -944,6 +944,7 @@ func New(options *Options) *API {
AllowBYOKSet: true,
AIBridgeTransportFactory: &api.AIBridgeTransportFactory,
AlwaysEnableDebugLogs: options.DeploymentValues.AI.Chat.DebugLoggingEnabled.Value(),
StreamSilenceTimeout: options.DeploymentValues.AI.Chat.StreamSilenceTimeout.Value(),
Experiments: experiments,
AgentConn: api.agentProvider.AgentConn,
AgentInactiveDisconnectTimeout: api.AgentInactiveDisconnectTimeout,
Expand Down
16 changes: 11 additions & 5 deletions coderd/x/chatd/chatd.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ type Server struct {
agentInactiveDisconnectTimeout time.Duration
dialTimeout time.Duration
instructionLookupTimeout time.Duration
streamSilenceTimeout time.Duration
createWorkspaceFn chattool.CreateWorkspaceFn
startWorkspaceFn chattool.StartWorkspaceFn
stopWorkspaceFn chattool.StopWorkspaceFn
Expand Down Expand Up @@ -3044,11 +3045,15 @@ type Config struct {
ReplicaID uuid.UUID
// StreamPartsDialer dials remote stream parts. Nil uses the local
// in-process channel dialer for every stream.
StreamPartsDialer StreamPartsDialer
PendingChatAcquireInterval time.Duration
MaxChatsPerAcquire int32
InFlightChatStaleAfter time.Duration
ChatHeartbeatInterval time.Duration
StreamPartsDialer StreamPartsDialer
PendingChatAcquireInterval time.Duration
MaxChatsPerAcquire int32
InFlightChatStaleAfter time.Duration
ChatHeartbeatInterval time.Duration
// StreamSilenceTimeout bounds how long a model attempt may go
// without receiving a stream part before it is canceled and
// retried. Zero uses the chatloop default.
StreamSilenceTimeout time.Duration
AgentConn AgentConnFunc
AgentInactiveDisconnectTimeout time.Duration
InstructionLookupTimeout time.Duration
Expand Down Expand Up @@ -3146,6 +3151,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
agentInactiveDisconnectTimeout: cfg.AgentInactiveDisconnectTimeout,
dialTimeout: defaultDialTimeout,
instructionLookupTimeout: instructionLookupTimeout,
streamSilenceTimeout: cfg.StreamSilenceTimeout,
createWorkspaceFn: cfg.CreateWorkspace,
startWorkspaceFn: cfg.StartWorkspace,
stopWorkspaceFn: cfg.StopWorkspace,
Expand Down
6 changes: 4 additions & 2 deletions coderd/x/chatd/chatloop/chatloop.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ import (
const (
// defaultStreamSilenceTimeout bounds how long an individual
// model attempt may go without receiving a stream part before
// the attempt is canceled and retried.
defaultStreamSilenceTimeout = 10 * time.Minute
// the attempt is canceled and retried. Five minutes tolerates
// long provider thinking pauses while bounding how long a
// wedged provider stream can stall a turn.
defaultStreamSilenceTimeout = 5 * time.Minute
streamSilenceGuardTimerTag = "streamSilenceGuard"
)

Expand Down
1 change: 1 addition & 0 deletions coderd/x/chatd/generation.go
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,7 @@ func (s *taskStarter) generateAssistant(
ProviderOptions: prepared.ProviderOptions,
PublishMessagePart: attempt.publish,
OnModelStreamStart: attempt.startModelInvocation,
StreamSilenceTimeout: s.server.streamSilenceTimeout,
Logger: s.opts.Logger,
Clock: s.opts.Clock,
Metrics: s.server.metrics,
Expand Down
27 changes: 20 additions & 7 deletions codersdk/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -4354,6 +4354,18 @@ Write out the current server config as YAML to stdout.`,
Group: &deploymentGroupChat,
YAML: "debugLoggingEnabled",
},
{
Name: "Chat: Stream Silence Timeout",
Description: "How long a chat model attempt may go without receiving a stream part before it is canceled and retried.",
Flag: "chat-stream-silence-timeout",
Env: "CODER_CHAT_STREAM_SILENCE_TIMEOUT",
Value: &c.AI.Chat.StreamSilenceTimeout,
Default: "5m",
Group: &deploymentGroupChat,
YAML: "streamSilenceTimeout",
Hidden: true, // Hidden because most operators should not need to modify this.
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
},
{
Name: "Chat: Hook URL",
Description: "HTTPS URL to receive chat agent lifecycle hook events (plain HTTP requires --chat-hook-allow-insecure). Hooks are disabled when unset. Requires the agent-lifecycle-hooks experiment.",
Expand Down Expand Up @@ -5092,13 +5104,14 @@ type AIBridgeProxyConfig struct {
}

type ChatConfig struct {
AcquireBatchSize serpent.Int64 `json:"acquire_batch_size" typescript:",notnull"`
DebugLoggingEnabled serpent.Bool `json:"debug_logging_enabled" typescript:",notnull"`
HookURL serpent.URL `json:"hook_url" typescript:",notnull"`
HookSecret serpent.String `json:"hook_secret" typescript:",notnull"`
HookTimeout serpent.Duration `json:"hook_timeout" typescript:",notnull"`
HookEnabled serpent.Bool `json:"hook_enabled" typescript:",notnull"`
HookAllowInsecure serpent.Bool `json:"hook_allow_insecure" typescript:",notnull"`
AcquireBatchSize serpent.Int64 `json:"acquire_batch_size" typescript:",notnull"`
DebugLoggingEnabled serpent.Bool `json:"debug_logging_enabled" typescript:",notnull"`
StreamSilenceTimeout serpent.Duration `json:"stream_silence_timeout" typescript:",notnull"`
HookURL serpent.URL `json:"hook_url" typescript:",notnull"`
HookSecret serpent.String `json:"hook_secret" typescript:",notnull"`
HookTimeout serpent.Duration `json:"hook_timeout" typescript:",notnull"`
HookEnabled serpent.Bool `json:"hook_enabled" typescript:",notnull"`
HookAllowInsecure serpent.Bool `json:"hook_allow_insecure" typescript:",notnull"`
// Deprecated: AI Gateway routing is now the only routing path. Setting this
// value has no effect. This option will be removed in a future release.
AIGatewayRoutingEnabled serpent.Bool `json:"ai_gateway_routing_enabled" typescript:",notnull" swaggerignore:"true"`
Expand Down
3 changes: 2 additions & 1 deletion docs/reference/api/general.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 18 additions & 13 deletions docs/reference/api/schemas.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions site/src/api/typesGenerated.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading