2021-08-11 19:09:14 +02:00
|
|
|
import hashlib
|
|
|
|
import json
|
|
|
|
import logging
|
2022-02-07 15:11:05 +01:00
|
|
|
from collections import Counter
|
|
|
|
|
2021-08-11 19:09:14 +02:00
|
|
|
import nagiosplugin
|
2021-08-13 11:00:43 +02:00
|
|
|
from typing import Iterable, Union
|
2021-08-11 19:09:14 +02:00
|
|
|
|
|
|
|
from .types import PatroniResource, ConnectionInfo, handle_unknown
|
|
|
|
|
|
|
|
_log = logging.getLogger("nagiosplugin")
|
|
|
|
|
|
|
|
|
|
|
|
def replace_chars(text: str) -> str:
|
|
|
|
return text.replace("'", "").replace(" ", "_")
|
|
|
|
|
|
|
|
|
|
|
|
class ClusterNodeCount(PatroniResource):
|
2021-08-12 11:38:55 +02:00
|
|
|
def probe(self: "ClusterNodeCount") -> Iterable[nagiosplugin.Metric]:
|
2021-08-11 19:09:14 +02:00
|
|
|
r = self.rest_api("cluster")
|
|
|
|
_log.debug(f"api call status: {r.status}")
|
|
|
|
_log.debug(f"api call data: {r.data}")
|
|
|
|
|
|
|
|
item_dict = json.loads(r.data)
|
2021-08-12 12:48:55 +02:00
|
|
|
role_counters: Counter[str] = Counter()
|
2021-08-11 19:09:14 +02:00
|
|
|
roles = []
|
2021-08-12 12:48:55 +02:00
|
|
|
status_counters: Counter[str] = Counter()
|
2021-08-11 19:09:14 +02:00
|
|
|
statuses = []
|
|
|
|
|
|
|
|
for member in item_dict["members"]:
|
|
|
|
roles.append(replace_chars(member["role"]))
|
|
|
|
statuses.append(replace_chars(member["state"]))
|
|
|
|
role_counters.update(roles)
|
|
|
|
status_counters.update(statuses)
|
|
|
|
|
|
|
|
# The actual check: members, running state
|
|
|
|
yield nagiosplugin.Metric("members", len(item_dict["members"]))
|
|
|
|
yield nagiosplugin.Metric("state_running", status_counters["running"])
|
|
|
|
|
|
|
|
# The performance data : role
|
|
|
|
for role in role_counters:
|
|
|
|
yield nagiosplugin.Metric(
|
2022-02-07 14:18:14 +01:00
|
|
|
f"role_{role}", role_counters[role], context="member_roles"
|
2021-08-11 19:09:14 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
# The performance data : statuses (except running)
|
|
|
|
for state in status_counters:
|
|
|
|
if state != "running":
|
|
|
|
yield nagiosplugin.Metric(
|
2022-02-07 14:18:14 +01:00
|
|
|
f"state_{state}", status_counters[state], context="member_statuses"
|
2021-08-11 19:09:14 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
class ClusterHasLeader(PatroniResource):
|
2021-08-12 11:38:55 +02:00
|
|
|
def probe(self: "ClusterHasLeader") -> Iterable[nagiosplugin.Metric]:
|
2021-08-11 19:09:14 +02:00
|
|
|
r = self.rest_api("cluster")
|
|
|
|
_log.debug(f"api call status: {r.status}")
|
|
|
|
_log.debug(f"api call data: {r.data}")
|
|
|
|
|
|
|
|
item_dict = json.loads(r.data)
|
|
|
|
is_leader_found = False
|
|
|
|
for member in item_dict["members"]:
|
|
|
|
if member["role"] == "leader" and member["state"] == "running":
|
|
|
|
is_leader_found = True
|
|
|
|
break
|
|
|
|
|
|
|
|
return [
|
|
|
|
nagiosplugin.Metric(
|
|
|
|
"has_leader",
|
|
|
|
1 if is_leader_found else 0,
|
|
|
|
)
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
class ClusterHasLeaderSummary(nagiosplugin.Summary):
|
|
|
|
def ok(self: "ClusterHasLeaderSummary", results: nagiosplugin.Result) -> str:
|
|
|
|
return "The cluster has a running leader."
|
|
|
|
|
|
|
|
@handle_unknown
|
|
|
|
def problem(self: "ClusterHasLeaderSummary", results: nagiosplugin.Result) -> str:
|
|
|
|
return "The cluster has no running leader."
|
|
|
|
|
|
|
|
|
|
|
|
class ClusterHasReplica(PatroniResource):
|
2021-08-13 11:00:43 +02:00
|
|
|
def __init__(
|
|
|
|
self: "ClusterHasReplica",
|
|
|
|
connection_info: ConnectionInfo,
|
|
|
|
max_lag: Union[int, None],
|
|
|
|
):
|
|
|
|
super().__init__(connection_info)
|
|
|
|
self.max_lag = max_lag
|
|
|
|
|
2021-08-12 11:38:55 +02:00
|
|
|
def probe(self: "ClusterHasReplica") -> Iterable[nagiosplugin.Metric]:
|
2021-08-11 19:09:14 +02:00
|
|
|
r = self.rest_api("cluster")
|
|
|
|
_log.debug(f"api call status: {r.status}")
|
|
|
|
_log.debug(f"api call data: {r.data}")
|
|
|
|
|
|
|
|
item_dict = json.loads(r.data)
|
|
|
|
replicas = []
|
2021-08-13 11:00:43 +02:00
|
|
|
healthy_replica = 0
|
|
|
|
unhealthy_replica = 0
|
2021-08-11 19:09:14 +02:00
|
|
|
for member in item_dict["members"]:
|
|
|
|
# FIXME are there other acceptable states
|
2021-08-13 11:00:43 +02:00
|
|
|
if member["role"] == "replica":
|
|
|
|
if member["state"] == "running" and member["lag"] != "unknown":
|
|
|
|
replicas.append({"name": member["name"], "lag": member["lag"]})
|
|
|
|
if self.max_lag is None or self.max_lag >= int(member["lag"]):
|
|
|
|
healthy_replica += 1
|
|
|
|
continue
|
|
|
|
unhealthy_replica += 1
|
2021-08-11 19:09:14 +02:00
|
|
|
|
|
|
|
# The actual check
|
2021-08-13 11:00:43 +02:00
|
|
|
yield nagiosplugin.Metric("healthy_replica", healthy_replica)
|
2021-08-11 19:09:14 +02:00
|
|
|
|
2022-02-07 14:18:14 +01:00
|
|
|
# The performance data : unhealthy replica count, replicas lag
|
2021-08-13 11:00:43 +02:00
|
|
|
yield nagiosplugin.Metric("unhealthy_replica", unhealthy_replica)
|
2021-08-11 19:09:14 +02:00
|
|
|
for replica in replicas:
|
|
|
|
yield nagiosplugin.Metric(
|
|
|
|
f"{replica['name']}_lag", replica["lag"], context="replica_lag"
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
# FIXME is this needed ??
|
|
|
|
# class ClusterHasReplicaSummary(nagiosplugin.Summary):
|
|
|
|
# def ok(self, results):
|
|
|
|
# def problem(self, results):
|
|
|
|
|
|
|
|
|
|
|
|
class ClusterConfigHasChanged(PatroniResource):
|
|
|
|
def __init__(
|
|
|
|
self: "ClusterConfigHasChanged",
|
|
|
|
connection_info: ConnectionInfo,
|
2022-02-07 15:01:50 +01:00
|
|
|
config_hash: str, # Always contains the old hash
|
2022-07-11 15:16:19 +02:00
|
|
|
state_file: str, # Only used to update the hash in the state_file (when needed)
|
2021-08-11 19:09:14 +02:00
|
|
|
):
|
|
|
|
super().__init__(connection_info)
|
|
|
|
self.state_file = state_file
|
|
|
|
self.config_hash = config_hash
|
|
|
|
|
2021-08-12 11:38:55 +02:00
|
|
|
def probe(self: "ClusterConfigHasChanged") -> Iterable[nagiosplugin.Metric]:
|
2021-08-11 19:09:14 +02:00
|
|
|
r = self.rest_api("config")
|
|
|
|
_log.debug(f"api call status: {r.status}")
|
|
|
|
_log.debug(f"api call data: {r.data}")
|
|
|
|
|
|
|
|
new_hash = hashlib.md5(r.data).hexdigest()
|
|
|
|
|
2022-02-07 15:01:50 +01:00
|
|
|
old_hash = self.config_hash
|
2021-08-11 19:09:14 +02:00
|
|
|
if self.state_file is not None:
|
2022-02-07 15:01:50 +01:00
|
|
|
_log.debug(f"Saving new hash to state file / cookie {self.state_file}")
|
2021-08-11 19:09:14 +02:00
|
|
|
cookie = nagiosplugin.Cookie(self.state_file)
|
|
|
|
cookie.open()
|
|
|
|
cookie["hash"] = new_hash
|
|
|
|
cookie.commit()
|
2022-02-07 15:01:50 +01:00
|
|
|
cookie.close()
|
2021-08-11 19:09:14 +02:00
|
|
|
|
|
|
|
_log.debug(f"hash info: old hash {old_hash}, new hash {new_hash}")
|
|
|
|
|
|
|
|
return [
|
|
|
|
nagiosplugin.Metric(
|
|
|
|
"is_configuration_changed",
|
|
|
|
1 if new_hash != old_hash else 0,
|
|
|
|
)
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
class ClusterConfigHasChangedSummary(nagiosplugin.Summary):
|
2022-02-07 15:01:50 +01:00
|
|
|
def __init__(self: "ClusterConfigHasChangedSummary", config_hash: str) -> None:
|
|
|
|
self.old_config_hash = config_hash
|
|
|
|
|
|
|
|
# Note: It would be helpful to display the old / new hash here. Unfortunately, it's not a metric.
|
|
|
|
# So we only have the old / expected one.
|
2021-08-11 19:09:14 +02:00
|
|
|
def ok(self: "ClusterConfigHasChangedSummary", results: nagiosplugin.Result) -> str:
|
2022-02-07 15:01:50 +01:00
|
|
|
return f"The hash of patroni's dynamic configuration has not changed ({self.old_config_hash})."
|
2021-08-11 19:09:14 +02:00
|
|
|
|
|
|
|
@handle_unknown
|
|
|
|
def problem(
|
|
|
|
self: "ClusterConfigHasChangedSummary", results: nagiosplugin.Result
|
|
|
|
) -> str:
|
2022-02-07 15:01:50 +01:00
|
|
|
return "The hash of patroni's dynamic configuration has changed. The old hash was {self.old_config_hash}."
|
2021-08-12 12:48:55 +02:00
|
|
|
|
|
|
|
|
|
|
|
class ClusterIsInMaintenance(PatroniResource):
|
|
|
|
def probe(self: "ClusterIsInMaintenance") -> Iterable[nagiosplugin.Metric]:
|
|
|
|
r = self.rest_api("cluster")
|
|
|
|
_log.debug(f"api call status: {r.status}")
|
|
|
|
_log.debug(f"api call data: {r.data}")
|
|
|
|
|
|
|
|
item_dict = json.loads(r.data)
|
|
|
|
|
|
|
|
# The actual check
|
2021-08-12 13:02:49 +02:00
|
|
|
return [
|
|
|
|
nagiosplugin.Metric(
|
|
|
|
"is_in_maintenance",
|
|
|
|
1 if "pause" in item_dict and item_dict["pause"] else 0,
|
|
|
|
)
|
|
|
|
]
|