Sign in

Process Grafana Alerts

There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.

Process Grafana Alert

problem = payload["alerts"][0]["annotations"]["description"] #{"alerts":[{"annotations":{"description":"Endpoint is Down: http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu in dagknows namespace.","summary":"Endpoint Down."},"dashboardURL":"","endsAt":"0001-01-01T00:00:00Z","fingerprint":"8b773b64aaeebd15","generatorURL":"http://localhost:3000/alerting/grafana/ee9l72z18oikgf/view?orgId=1","labels":{"alertname":"ProbeFailing-MinikubeApplication (copy)","grafana_folder":"alert_rule_folder_1m","instance":"http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu","job":"blackbox-exporter-metrics"},"panelURL":"","silenceURL":"http://localhost:3000/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DProbeFailing-MinikubeApplication+%28copy%29&matcher=grafana_folder%3Dalert_rule_folder_1m&matcher=instance%3Dhttp%3A%2F%2Fdemo.dagknows.com%3A8081%2Fapi%2Ftasks%2FZ6ylcZTzO1xURkNUngRu&matcher=job%3Dblackbox-exporter-metrics&orgId=1","startsAt":"2025-06-02T01:59:20Z","status":"firing","valueString":"[ var='A' labels={instance=http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu, job=blackbox-exporter-metrics} value=0 ], [ var='C' labels={instance=http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu, job=blackbox-exporter-metrics} value=1 ]","values":{"A":0,"C":1}}],"commonAnnotations":{"description":"Endpoint is Down: http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu in dagknows namespace.","summary":"Endpoint Down."},"commonLabels":{"alertname":"ProbeFailing-MinikubeApplication (copy)","grafana_folder":"alert_rule_folder_1m","instance":"http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu","job":"blackbox-exporter-metrics"},"externalURL":"http://localhost:3000/","groupKey":"{}/{__grafana_autogenerated__=\"true\"}/{__grafana_receiver__=\"dk_webhook_security_group_demo\"}/{__grafana_route_settings_hash__=\"31cd9e4f79dd0eae\"}:{alertname=\"ProbeFailing-MinikubeApplication (copy)\", grafana_folder=\"alert_rule_folder_1m\"}","groupLabels":{"alertname":"ProbeFailing-MinikubeApplication (copy)","grafana_folder":"alert_rule_folder_1m"},"message":"**Firing**\n\nValue: A=0, C=1\nLabels:\n - alertname = ProbeFailing-MinikubeApplication (copy)\n - grafana_folder = alert_rule_folder_1m\n - instance = http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu\n - job = blackbox-exporter-metrics\nAnnotations:\n - description = Endpoint is Down: http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu in dagknows namespace.\n - summary = Endpoint Down.\nSource: http://localhost:3000/alerting/grafana/ee9l72z18oikgf/view?orgId=1\nSilence: http://localhost:3000/alerting/silence/new?alertmanager=grafana&matcher=alertname%3DProbeFailing-MinikubeApplication+%28copy%29&matcher=grafana_folder%3Dalert_rule_folder_1m&matcher=instance%3Dhttp%3A%2F%2Fdemo.dagknows.com%3A8081%2Fapi%2Ftasks%2FZ6ylcZTzO1xURkNUngRu&matcher=job%3Dblackbox-exporter-metrics&orgId=1\n","orgId":1,"receiver":"dk_webhook_security_group_demo","state":"alerting","status":"firing","title":"[FIRING:1] ProbeFailing-MinikubeApplication (copy) alert_rule_folder_1m (http://demo.dagknows.com:8081/api/tasks/Z6ylcZTzO1xURkNUngRu blackbox-exporter-metrics)","truncatedAlerts":0,"version":"1"}
copied
  1. 1

    Choose and execute a runbook for a given problem

    There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
    1
    1. 1.1

      Search runbooks for a given problem

      There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
      import json import copy cred_label = "dktoken" search_url = f"https://dev.dagknows.com/api/tasks/?with_pending_perms=false&page_key=0&page_size=10&q={problem}&knn.k=3&knn.nc=10&order_by=elastic&tags=tshoot" op = _rest_api("get", search_url, "", {}, "", cred_label) resp = op.json() tmptasks = [] for task in resp["tasks"]: tmptask = copy.deepcopy(task) for key, value in task.items(): if key not in ["title", "description", "input_params", "output_params", "id", "tags"]: tmptask.pop(key, None) tmptasks.append(tmptask) print(json.dumps(tmptasks, indent=4, default=str)) # print(json.dumps(op, indent=4)) relevant_runbooks = tmptasks
      copied
      1.1
    2. 1.2

      Send the problem and the relevant runbooks to LLM and get the runbook to execute

      There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
      import json import openai # Get the API key from the environment variable api_key = getEnvVar('OPENAI_API_KEY') # Initialize the OpenAI client with the API key openai.api_key = api_key system_prompt = """ You are a DevOps, ITOps, and SRE expert who excells at picking the right tool to solve a problem. User will give you a problem to be solved along with an array of tools available. Your task is to identify the right tool to execute along with the input values required for execution. The list of tools is formatted as follows: [ { "id" : "<A unique ID of the tool>" "title": "<A one line title that tells what this tool does>", "description": " Description of the tool", "input_params": [ # List of input parameters the tool accepts, each parameter is a dictionary { "default_value": "<Optional. Default value of the >", "description": "<Optional. Description of the input parameter>", "name": "<name of the input parameter>", "param_type": "<Python type of the input parameter>", "required": <Boolean indicating if the input value is required or not> } ] "output_params": [ # List of output parameters the tool produces, Each parameter is a dictionary just like input parameter dict. { <Same dictionary as above for an input parameter> } ] }, ] Your output should be strictly a JSON as follows: { "id" : "<ID of the tool to execute. If no suitable tool found then set it to null.>", "input_params" : [ # List of input parameters with values to assign for execution. Make sure required ones are there. { "name" : "<Name of the param to assign value to>", "value" : "<The value to be assigned to the input. Make sure it matches the input type>" } ] } Do not give any preamble or explanation. Just JSON. """ user_prompt = f""" Here's the problem to solve: {problem} Here's the list of available tools: {json.dumps(relevant_runbooks, indent=4)} """ messages = [ {"role" : "system", "content" : system_prompt}, {"role" : "user", "content" : user_prompt} ] response = openai.chat.completions.create( model='gpt-4o', messages=messages, temperature=0 ) raw_content = response.choices[0].message.content cleaned_output = "\n".join( [line for line in raw_content.split("\n") if not line.startswith("```")] ).strip() chosen_tool = json.loads(cleaned_output) print(chosen_tool)
      copied
      1.2
    3. 1.3

      Execute the chosen runbook

      There was a problem that the LLM was not able to address. Please rephrase your prompt and try again.
      cred_label = "dktoken" api_endpoint = f"https://dev.dagknows.com/api/tasks/{chosen_tool["id"]}/execute" param_values = {} for iparam in chosen_tool["input_params"]: name = iparam["name"] value = iparam["value"] param_values[name] = value job = { "proxy_alias": "dev", "param_values": param_values, "special_param_values": { "stop_after_first_failure": "", "schedule_options": { "start": None, "repeat_interval": "", "repeat_interval_type": "", "end_option": "", "end": None, "num_iterations": "" }, "wsid": "", "proxy_alias": "dev", "button": "regular" }, "output_params": {}, "runbook_task_id": chosen_tool["id"], "starting_child_path": "", "conv_id": f"tconv_{chosen_tool["id"]}", "role": "", "experimental": {} } body = {} body["job"] = job print("API endpoint: ", api_endpoint) print("Body: ", json.dumps(body, indent=4)) op = _rest_api("post", api_endpoint, "", {}, body, cred_label).json() print(json.dumps(op, indent=4, default=str)) job_id = op["job"]["job_id"] job_url = f"https://dev.dagknows.com/tasks/{chosen_tool['id']}?job_id={job_id}&iter=0" task_title = op["job"]["title"] table = context.newtable() # This creates a new table table.num_rows = 2 table.num_cols = 4 table.title = "Chosen runbook" table.has_header_row = True # If the first row is header row then set to True else False table.setval(0, 0, "ID") table.setval(0, 1, "Title") table.setval(0, 2, "Params") table.setval(0, 3, "Execution results") table.setval(1, 0, chosen_tool["id"]) table.setval(1, 1, task_title) table.setval(1, 2, json.dumps(param_values, indent=4, default=str)) table.setval(1, 3, f"<a href={job_url}>Link</a>")
      copied
      1.3