Resolve Incident
curl --request POST \
--url {scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes."
}
'import requests
url = "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload = { "resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes." }
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({resolution: 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'})
};
fetch('{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_PORT => "62437",
CURLOPT_URL => "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'resolution' => 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload := strings.NewReader("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
http = Net::HTTP.new(url.host, url.port)
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}"
response = http.request(request)
puts response.read_body{
"apiVersion": "v1",
"kind": "Incident",
"spec": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"status": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"resourceMeta": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"uid": "5b2d7c1e-3f4a-4e8b-9c6d-1a2b3c4d5e6f",
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
}
}
Incidents
Resolver Incidente
Resolve um incidente manualmente — usado quando a plataforma escala para ação humana ou quando um operador confirma que o problema foi corrigido
POST
/
incidents
/
{name}
/
resolve
Resolve Incident
curl --request POST \
--url {scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes."
}
'import requests
url = "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload = { "resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes." }
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({resolution: 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'})
};
fetch('{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_PORT => "62437",
CURLOPT_URL => "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'resolution' => 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload := strings.NewReader("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
http = Net::HTTP.new(url.host, url.port)
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}"
response = http.request(request)
puts response.read_body{
"apiVersion": "v1",
"kind": "Incident",
"spec": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"status": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"resourceMeta": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"uid": "5b2d7c1e-3f4a-4e8b-9c6d-1a2b3c4d5e6f",
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
}
}
string
obrigatório
Nome do recurso Issue (ex.:
payment-service-oom-kill-1773933600)string
Namespace do incidente. Se omitido, todos os namespaces são pesquisados e o primeiro incidente com esse nome é resolvido — informe-o explicitamente quando nomes puderem se repetir entre namespaces.
string
Descrição de como o incidente foi resolvido. Se omitido (ou com corpo vazio), o padrão é
"Manually resolved via dashboard".Boa prática: sempre informe uma mensagem de resolução significativa para a trilha de auditoria.Quando Usar Este Endpoint
1. Incidentes Escalados (Ação Humana Necessária)
Quando a remediação automática se esgota —maxRemediationAttempts tentativas (spec.aiops.maxRemediationAttempts da Instance, padrão 5, ou o maxAttempts do Runbook correspondente) — o incidente vai para Escalated. A partir daí:
- As políticas de notificação e escalonamento configuradas alertam o time de plantão
- Nenhuma remediação automática adicional é tentada
- Se o auto-resolve estiver ligado (
spec.aiops.enableAutoResolveda Instance, padrãotrue), o operator verifica o recurso a cada 30 segundos e resolve o incidente sozinho assim que todas as réplicas estiverem saudáveis - Caso contrário, o incidente fica
Escalatedaté alguém resolvê-lo por este endpoint ou pelo dashboard web
2. Verificação Manual Após Auto-Remediação
Mesmo quando a remediação automática funciona, operadores podem querer fechar o incidente com o próprio texto de resolução.3. Falsos Positivos
Quando o incidente se revela um falso positivo, resolva-o com uma resolução como"Falso positivo — pico de métrica causado por job batch agendado".
Fluxo de Resolução para Incidentes Escalados
Incidente Detectado
↓
Análise IA → Remediação Runbook/Agêntica
↓ ↓
Sucesso → Resolved Falha (retry até maxRemediationAttempts)
↓
Tentativas esgotadas
↓
ESCALATED ← Você está aqui
↓
┌─────────────────┼──────────────────┐
↓ ↓ ↓
Resolve via API Botão no Dashboard Auto-resolve
POST /resolve "Resolver" (recurso saudável)
↓ ↓ ↓
└─────────────────┼──────────────────┘
↓
Estado: Resolved
O Que Acontece na Resolução
- Mudança de estado: qualquer estado exceto
Resolved(incluindoEscalated,Failede estados em andamento) viraResolved;status.resolutionestatus.resolvedAtsão preenchidos. Resolver um incidente já resolvido retorna409. - Annotations adicionadas:
aiops.chatcli.io/resolved-by— a role da API key que chamou (operatorouadmin)aiops.chatcli.io/resolved-at— horário da resoluçãoaiops.chatcli.io/manual-resolution—"true"
Resolvedé terminal: o controller de Issue para de processar o incidente.
Uma resolução manual não: gera PostMortem (PostMortems só são criados quando um RemediationPlan termina), registra AuditEvent, interrompe um RemediationPlan já em execução, nem limpa a entrada de deduplicação do watcher para o recurso (o gancho existe, mas não está ligado no operator), então uma anomalia recorrente no mesmo recurso pode continuar deduplicada até a entrada expirar (
spec.aiops.dedupTTLMinutes da Instance, padrão 30).{
"apiVersion": "v1",
"kind": "Incident",
"spec": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"status": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"resourceMeta": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"uid": "5b2d7c1e-3f4a-4e8b-9c6d-1a2b3c4d5e6f",
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
}
}
{
"apiVersion": "v1",
"kind": "Error",
"error": "Not Found",
"code": 404,
"message": "issue not found: issues.platform.chatcli.io \"payment-service-oom-kill-1773000000\" not found"
}
{
"apiVersion": "v1",
"kind": "Error",
"error": "Conflict",
"code": 409,
"message": "issue is already resolved"
}
Autorizações
API key sent in the X-API-Key header. Keys are read from the Secret chatcli-operator-secrets, key api-keys (fallback: ConfigMap chatcli-operator-config, key api-keys) in the operator namespace, as a YAML list of {key, role, name, description} (name is the identity recorded on approval decisions); the operator chart creates it only with apiKeys.create: true. Roles: viewer < operator < admin — any other role string is denied everywhere. Changes are picked up within about 30 seconds; deleting both the Secret and the ConfigMap revokes every key. With no keys configured every /api/ call returns 401, unless CHATCLI_OPERATOR_DEV_MODE=true, which grants admin without a key (development only).
Parâmetros de caminho
Issue name.
Exemplo:
"payment-service-oom-1710860400"
Parâmetros de consulta
Namespace of the object. When omitted, all namespaces are searched and the first object with this name is used.
Corpo
application/json
How the incident was resolved. Defaults to Manually resolved via dashboard when omitted.