Resolve Incident
curl --request POST \
--url {scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes."
}
'import requests
url = "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload = { "resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes." }
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({resolution: 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'})
};
fetch('{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_PORT => "62437",
CURLOPT_URL => "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'resolution' => 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload := strings.NewReader("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
http = Net::HTTP.new(url.host, url.port)
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}"
response = http.request(request)
puts response.read_body{
"apiVersion": "v1",
"kind": "Incident",
"spec": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"status": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"resourceMeta": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"uid": "5b2d7c1e-3f4a-4e8b-9c6d-1a2b3c4d5e6f",
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
}
}
Incidents
Resolve Incident
Manually resolves an incident — used when the platform escalates to human action or when an operator confirms the issue is fixed
POST
/
incidents
/
{name}
/
resolve
Resolve Incident
curl --request POST \
--url {scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes."
}
'import requests
url = "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload = { "resolution": "Hotfix v2.4.1 deployed; memory stable for 15 minutes." }
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({resolution: 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'})
};
fetch('{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_PORT => "62437",
CURLOPT_URL => "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'resolution' => 'Hotfix v2.4.1 deployed; memory stable for 15 minutes.'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve"
payload := strings.NewReader("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("{scheme}://{host}:{port}/{basePath}/incidents/{name}/resolve")
http = Net::HTTP.new(url.host, url.port)
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"resolution\": \"Hotfix v2.4.1 deployed; memory stable for 15 minutes.\"\n}"
response = http.request(request)
puts response.read_body{
"apiVersion": "v1",
"kind": "Incident",
"spec": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"status": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"resourceMeta": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"uid": "5b2d7c1e-3f4a-4e8b-9c6d-1a2b3c4d5e6f",
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
}
}
string
required
Name of the Issue resource (e.g.,
payment-service-oom-kill-1773933600)string
Namespace of the incident. When omitted, every namespace is searched and the first incident with that name is resolved — pass it explicitly when names can repeat across namespaces.
string
Description of how the incident was resolved. If omitted (or the body is empty), defaults to
"Manually resolved via dashboard".Best practice: always provide a meaningful resolution message for the audit trail.When to Use This Endpoint
1. Escalated Incidents (Human Action Required)
When automatic remediation is exhausted —maxRemediationAttempts attempts (Instance spec.aiops.maxRemediationAttempts, default 5, or the matching Runbook’s maxAttempts) — the incident moves to Escalated. From there:
- Configured notification and escalation policies alert the on-call team
- No further automatic remediation is attempted
- If auto-resolve is enabled (Instance
spec.aiops.enableAutoResolve, defaulttrue), the operator re-checks the resource every 30 seconds and resolves the incident itself once all replicas are healthy - Otherwise the incident stays
Escalateduntil someone resolves it through this endpoint or the web dashboard
2. Manual Verification After Auto-Remediation
Even when automatic remediation succeeds, operators may want to close the incident with their own resolution text.3. False Positives
When an incident turns out to be a false positive, resolve it with a resolution such as"False positive — metric spike caused by scheduled batch job".
Resolution Flow for Escalated Incidents
Incident Detected
↓
AI Analysis → Runbook/Agentic Remediation
↓ ↓
Success → Resolved Failure (retry up to maxRemediationAttempts)
↓
All attempts exhausted
↓
ESCALATED ← You are here
↓
┌─────────────────┼──────────────────┐
↓ ↓ ↓
API Resolve Dashboard button Auto-resolve
POST /resolve "Resolve" (resource healthy)
↓ ↓ ↓
└─────────────────┼──────────────────┘
↓
State: Resolved
What Happens on Resolution
- State change: any state except
Resolved(includingEscalated,Failedand in-progress states) becomesResolved;status.resolutionandstatus.resolvedAtare set. Resolving an already resolved incident returns409. - Annotations added:
aiops.chatcli.io/resolved-by— the role of the calling API key (operatororadmin)aiops.chatcli.io/resolved-at— resolution timestampaiops.chatcli.io/manual-resolution—"true"
Resolvedis terminal: the Issue controller stops processing the incident.
A manual resolve does not: generate a PostMortem (PostMortems are created only when a RemediationPlan finishes), record an AuditEvent, stop a RemediationPlan that is already running, or clear the watcher’s deduplication entry for the resource (the hook exists but is not wired in the operator), so a recurring anomaly on the same resource can stay deduplicated until the entry expires (Instance
spec.aiops.dedupTTLMinutes, default 30).{
"apiVersion": "v1",
"kind": "Incident",
"spec": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"status": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"severity": "high",
"source": "watcher",
"signalType": "oom_kill",
"resource": {
"kind": "Deployment",
"name": "payment-service",
"namespace": "production"
},
"description": "Auto-detected oom_kill anomaly: container restarted after OOMKill (value=5, threshold=3)",
"riskScore": 65,
"state": "Resolved",
"detectedAt": "2026-03-19T15:20:00Z",
"resolvedAt": "2026-03-19T16:45:00Z",
"resolution": "Memory leak fixed in payment-service v2.4.1 hotfix, deployed manually. Stable for 15 minutes.",
"remediationAttempts": 5,
"maxRemediationAttempts": 5,
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
},
"resourceMeta": {
"name": "payment-service-oom-kill-1773933600",
"namespace": "production",
"uid": "5b2d7c1e-3f4a-4e8b-9c6d-1a2b3c4d5e6f",
"creationTimestamp": "2026-03-19T15:20:00Z",
"labels": {
"platform.chatcli.io/inc-id": "INC-20260319-001",
"platform.chatcli.io/resource": "payment-service",
"platform.chatcli.io/signal": "oom_kill"
},
"annotations": {
"aiops.chatcli.io/manual-resolution": "true",
"aiops.chatcli.io/resolved-at": "2026-03-19T16:45:00Z",
"aiops.chatcli.io/resolved-by": "operator"
}
}
}
{
"apiVersion": "v1",
"kind": "Error",
"error": "Not Found",
"code": 404,
"message": "issue not found: issues.platform.chatcli.io \"payment-service-oom-kill-1773000000\" not found"
}
{
"apiVersion": "v1",
"kind": "Error",
"error": "Conflict",
"code": 409,
"message": "issue is already resolved"
}
Authorizations
API key sent in the X-API-Key header. Keys are read from the Secret chatcli-operator-secrets, key api-keys (fallback: ConfigMap chatcli-operator-config, key api-keys) in the operator namespace, as a YAML list of {key, role, name, description} (name is the identity recorded on approval decisions); the operator chart creates it only with apiKeys.create: true. Roles: viewer < operator < admin — any other role string is denied everywhere. Changes are picked up within about 30 seconds; deleting both the Secret and the ConfigMap revokes every key. With no keys configured every /api/ call returns 401, unless CHATCLI_OPERATOR_DEV_MODE=true, which grants admin without a key (development only).
Path Parameters
Issue name.
Example:
"payment-service-oom-1710860400"
Query Parameters
Namespace of the object. When omitted, all namespaces are searched and the first object with this name is used.
Body
application/json
How the incident was resolved. Defaults to Manually resolved via dashboard when omitted.