Process Grafana Alert

problem = payload["alerts"][0]["annotations"]["description"] #{"alerts":[{"annotations":{"description":"Endpoint is Down: http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu in dagknows namespace.","summary":"Endpoint Down."},"dashboardURL":"","endsAt":"0001-01-01T00:00:00Z","fingerprint":"8b773b64aaeebd15","generatorURL":"http://localhost:3000/alerting/grafana/ee9l72z18oikgf/view?orgId=1","labels":{"alertname":"ProbeFailing-MinikubeApplication (copy)","grafana_folder":"alert_rule_folder_1m","instance":"http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu","job":"blackbox-exporter-metrics"},"panelURL":"","silenceURL":"http://localhost:3000/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DProbeFailing-MinikubeApplication+%28copy%29&matcher=grafana_folder%3Dalert_rule_folder_1m&matcher=instance%3Dhttp%3A%2F%2Fdemo.dagknows.com%3A8081%2Fapi%2Ftasks%2FZ6ylcZTzO1xURkNUngRu&matcher=job%3Dblackbox-exporter-metrics&orgId=1","startsAt":"2025-06-02T01:59:20Z","status":"firing","valueString":"[ var='A' labels={instance=http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu, job=blackbox-exporter-metrics} value=0 ], [ var='C' labels={instance=http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu, job=blackbox-exporter-metrics} value=1 ]","values":{"A":0,"C":1}}],"commonAnnotations":{"description":"Endpoint is Down: http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu in dagknows namespace.","summary":"Endpoint Down."},"commonLabels":{"alertname":"ProbeFailing-MinikubeApplication (copy)","grafana_folder":"alert_rule_folder_1m","instance":"http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu","job":"blackbox-exporter-metrics"},"externalURL":"http://localhost:3000/","groupKey":"{}/{__grafana_autogenerated__=\"true\"}/{__grafana_receiver__=\"dk_webhook_security_group_demo\"}/{__grafana_route_settings_hash__=\"31cd9e4f79dd0eae\"}:{alertname=\"ProbeFailing-MinikubeApplication (copy)\", grafana_folder=\"alert_rule_folder_1m\"}","groupLabels":{"alertname":"ProbeFailing-MinikubeApplication (copy)","grafana_folder":"alert_rule_folder_1m"},"message":"**Firing**\n\nValue: A=0, C=1\nLabels:\n - alertname = ProbeFailing-MinikubeApplication (copy)\n - grafana_folder = alert_rule_folder_1m\n - instance = http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu\n - job = blackbox-exporter-metrics\nAnnotations:\n - description = Endpoint is Down: http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu in dagknows namespace.\n - summary = Endpoint Down.\nSource: http://localhost:3000/alerting/grafana/ee9l72z18oikgf/view?orgId=1\nSilence: http://localhost:3000/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DProbeFailing-MinikubeApplication+%28copy%29&matcher=grafana_folder%3Dalert_rule_folder_1m&matcher=instance%3Dhttp%3A%2F%2Fdemo.dagknows.com%3A8081%2Fapi%2Ftasks%2FZ6ylcZTzO1xURkNUngRu&matcher=job%3Dblackbox-exporter-metrics&orgId=1\n","orgId":1,"receiver":"dk_webhook_security_group_demo","state":"alerting","status":"firing","title":"[FIRING:1] ProbeFailing-MinikubeApplication (copy) alert_rule_folder_1m (http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu blackbox-exporter-metrics)","truncatedAlerts":0,"version":"1"}

copied

Demo Grafana

1IUkBwsZXofY0OwP4xSAsChoose and execute a runbook for a given problem
1
Choose and execute a runbook for a given problem
There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
inputs
required
outputs
1
1.1SC7PHq17Ww7QXoJmcdMZSearch runbooks for a given problem
1.1
Search runbooks for a given problem
There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
inputs
required
outputs
import json import copy cred_label = "dktoken" search_url = f"https://dev.dagknows.com/api/tasks/?with_pending_perms=false&page_key=0&page_size=10&q={problem}&knn.k=3&knn.nc=10&order_by=elastic&tags=tshoot" op = _rest_api("get", search_url, "", {}, "", cred_label) resp = op.json() tmptasks = [] for task in resp["tasks"]: tmptask = copy.deepcopy(task) for key, value in task.items(): if key not in ["title", "description", "input_params", "output_params", "id", "tags"]: tmptask.pop(key, None) tmptasks.append(tmptask) print(json.dumps(tmptasks, indent=4, default=str)) # print(json.dumps(op, indent=4)) relevant_runbooks = tmptasks
copied
1.1
1.2wIRFKmUodwg7ewunpsmT
1.2
Send the problem and the relevant runbooks to LLM and get the runbook to execute
There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
inputs
required
required
outputs
import json import openai # Get the API key from the environment variable api_key = getEnvVar('OPENAI_API_KEY') # Initialize the OpenAI client with the API key openai.api_key = api_key system_prompt = """ You are a DevOps, ITOps, and SRE expert who excells at picking the right tool to solve a problem. User will give you a problem to be solved along with an array of tools available. Your task is to identify the right tool to execute along with the input values required for execution. The list of tools is formatted as follows: [ { "id" : "<A unique ID of the tool>" "title": "<A one line title that tells what this tool does>", "description": " Description of the tool", "input_params": [ # List of input parameters the tool accepts, each parameter is a dictionary { "default_value": "<Optional. Default value of the >", "description": "<Optional. Description of the input parameter>", "name": "<name of the input parameter>", "param_type": "<Python type of the input parameter>", "required": <Boolean indicating if the input value is required or not> } ] "output_params": [ # List of output parameters the tool produces, Each parameter is a dictionary just like input parameter dict. { <Same dictionary as above for an input parameter> } ] }, ] Your output should be strictly a JSON as follows: { "id" : "<ID of the tool to execute. If no suitable tool found then set it to null.>", "input_params" : [ # List of input parameters with values to assign for execution. Make sure required ones are there. { "name" : "<Name of the param to assign value to>", "value" : "<The value to be assigned to the input. Make sure it matches the input type>" } ] } Do not give any preamble or explanation. Just JSON. """ user_prompt = f""" Here's the problem to solve: {problem} Here's the list of available tools: {json.dumps(relevant_runbooks, indent=4)} """ messages = [ {"role" : "system", "content" : system_prompt}, {"role" : "user", "content" : user_prompt} ] response = openai.chat.completions.create( model='gpt-4o', messages=messages, temperature=0 ) raw_content = response.choices[0].message.content cleaned_output = "\n".join( [line for line in raw_content.split("\n") if not line.startswith("```")] ).strip() chosen_tool = json.loads(cleaned_output) print(chosen_tool)
copied
1.2
1.3WfZtepDNEJFpiaRIlCO3Execute the chosen runbook
1.3
Execute the chosen runbook
There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
inputs
required
outputs
cred_label = "dktoken" api_endpoint = f"https://dev.dagknows.com/api/tasks/{chosen_tool["id"]}/execute" param_values = {} for iparam in chosen_tool["input_params"]: name = iparam["name"] value = iparam["value"] param_values[name] = value job = { "proxy_alias": "dev", "param_values": param_values, "special_param_values": { "stop_after_first_failure": "", "schedule_options": { "start": None, "repeat_interval": "", "repeat_interval_type": "", "end_option": "", "end": None, "num_iterations": "" }, "wsid": "", "proxy_alias": "dev", "button": "regular" }, "output_params": {}, "runbook_task_id": chosen_tool["id"], "starting_child_path": "", "conv_id": f"tconv_{chosen_tool["id"]}", "role": "", "experimental": {} } body = {} body["job"] = job print("API endpoint: ", api_endpoint) print("Body: ", json.dumps(body, indent=4)) op = _rest_api("post", api_endpoint, "", {}, body, cred_label).json() print(json.dumps(op, indent=4, default=str)) job_id = op["job"]["job_id"] job_url = f"https://dev.dagknows.com/tasks/{chosen_tool['id']}?job_id={job_id}&iter=0" task_title = op["job"]["title"] table = context.newtable() # This creates a new table table.num_rows = 2 table.num_cols = 4 table.title = "Chosen runbook" table.has_header_row = True # If the first row is header row then set to True else False table.setval(0, 0, "ID") table.setval(0, 1, "Title") table.setval(0, 2, "Params") table.setval(0, 3, "Execution results") table.setval(1, 0, chosen_tool["id"]) table.setval(1, 1, task_title) table.setval(1, 2, json.dumps(param_values, indent=4, default=str)) table.setval(1, 3, f"<a href={job_url}>Link</a>")
copied
1.3

Process Grafana Alerts

Resources

Company

Legal