Private
Public Access
1
0

style: cargo fmt --all to fix CI format check
Some checks failed
CI Pipeline / Rust Format Check (push) Successful in 5s
CI Pipeline / Clippy Lints (push) Successful in 45s
CI Pipeline / Rust Unit Tests (push) Successful in 1m2s
CI Pipeline / Security Audit (push) Successful in 4s
CI Pipeline / Frontend Lint & Type Check (push) Failing after 10s
CI Pipeline / Build .deb & Release (push) Has been skipped

This commit is contained in:
2026-05-06 03:57:01 +00:00
parent 3d9b2d4917
commit 12d640e5de
12 changed files with 199 additions and 137 deletions

View File

@ -223,15 +223,18 @@ impl AgentClient {
/// `GET /api/v1/system/services/{name}` — check status of a specific service on the agent. /// `GET /api/v1/system/services/{name}` — check status of a specific service on the agent.
#[instrument(skip(self), fields(base_url = %self.base_url, service_name = %service_name))] #[instrument(skip(self), fields(base_url = %self.base_url, service_name = %service_name))]
pub async fn service_status(&self, service_name: &str) -> Result<ServiceStatusData, AgentClientError> { pub async fn service_status(
self.get(&format!("system/services/{}", service_name), &[]).await &self,
service_name: &str,
) -> Result<ServiceStatusData, AgentClientError> {
self.get(&format!("system/services/{}", service_name), &[])
.await
} }
// -------------------------------------------------------- // --------------------------------------------------------
// Private POST helper // Private POST helper
// -------------------------------------------------------- // --------------------------------------------------------
/// Execute a POST request against `{base_url}/{path}`, serialize `body` as /// Execute a POST request against `{base_url}/{path}`, serialize `body` as
/// JSON, deserialize the [`AgentEnvelope`], and extract the `data` field — /// JSON, deserialize the [`AgentEnvelope`], and extract the `data` field —
/// or propagate an [`AgentClientError::ApiError`]. /// or propagate an [`AgentClientError::ApiError`].

View File

@ -301,8 +301,9 @@ impl CertAuthority {
); );
// Also issue a server certificate for the agent's TLS listener. // Also issue a server certificate for the agent's TLS listener.
let (server_cert_pem, server_key_pem, server_serial_number, _server_expires_at) = let (server_cert_pem, server_key_pem, server_serial_number, _server_expires_at) = self
self.issue_server_cert(host_id, hostname, ip_address, db).await?; .issue_server_cert(host_id, hostname, ip_address, db)
.await?;
Ok(IssuedCert { Ok(IssuedCert {
cert_pem, cert_pem,

View File

@ -287,7 +287,10 @@ pub async fn verify_integrity(pool: &PgPool) -> IntegrityResult {
.unwrap_or_default(); .unwrap_or_default();
let ip_str = row.ip_address.as_deref().unwrap_or(""); let ip_str = row.ip_address.as_deref().unwrap_or("");
let rid = row.request_id.as_deref().unwrap_or(""); let rid = row.request_id.as_deref().unwrap_or("");
let created_str = row.created_at.map(|c| c.to_rfc3339_opts(chrono::SecondsFormat::Micros, true)).unwrap_or_default(); let created_str = row
.created_at
.map(|c| c.to_rfc3339_opts(chrono::SecondsFormat::Micros, true))
.unwrap_or_default();
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
hasher.update(row.prev_hash.as_bytes()); hasher.update(row.prev_hash.as_bytes());

View File

@ -101,7 +101,9 @@ impl AppConfig {
} }
} }
fn default_health_check_poll_interval() -> u64 { 300 } fn default_health_check_poll_interval() -> u64 {
300
}
impl Default for AppConfig { impl Default for AppConfig {
fn default() -> Self { fn default() -> Self {

View File

@ -9,7 +9,7 @@ pub mod request_id;
// Re-export commonly used types // Re-export commonly used types
pub use config::AppConfig; pub use config::AppConfig;
pub use crypto::{CryptoError, KEY_PATH, decrypt, encrypt, load_or_create_key}; pub use crypto::{decrypt, encrypt, load_or_create_key, CryptoError, KEY_PATH};
pub use error::{AppError, ErrorResponse}; pub use error::{AppError, ErrorResponse};
pub use models::{ pub use models::{
AuthProvider, CreateGroupRequest, CreateHealthCheckRequest, CreateHostRequest, AuthProvider, CreateGroupRequest, CreateHealthCheckRequest, CreateHostRequest,

View File

@ -189,7 +189,10 @@ pub fn build_router(state: AppState) -> Router {
.merge(routes::ws::ticket_router()) .merge(routes::ws::ticket_router())
// Reports // Reports
.nest("/reports", routes::reports::router()) .nest("/reports", routes::reports::router())
.nest("/hosts/{host_id}/health-checks", routes::health_checks::router()) .nest(
"/hosts/{host_id}/health-checks",
routes::health_checks::router(),
)
// Settings (admin-only) // Settings (admin-only)
.nest("/settings", routes::settings::router()) .nest("/settings", routes::settings::router())
// Apply auth middleware to all the above // Apply auth middleware to all the above

View File

@ -23,11 +23,11 @@ use pm_core::{
UpdateHealthCheckRequest, UpdateHealthCheckRequest,
}, },
}; };
use reqwest::tls::Version;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{json, Value}; use serde_json::{json, Value};
use std::path::PathBuf; use std::path::PathBuf;
use uuid::Uuid; use uuid::Uuid;
use reqwest::tls::Version;
use crate::AppState; use crate::AppState;
@ -85,9 +85,8 @@ async fn operator_can_access_host(
} }
// Also allow if host has no groups (ungrouped) // Also allow if host has no groups (ungrouped)
let has_groups: bool = sqlx::query_scalar( let has_groups: bool =
"SELECT EXISTS (SELECT 1 FROM host_groups WHERE host_id = $1)", sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM host_groups WHERE host_id = $1)")
)
.bind(host_id) .bind(host_id)
.fetch_one(pool) .fetch_one(pool)
.await .await
@ -117,15 +116,15 @@ async fn list_health_checks(
if !can_access { if !can_access {
return Err(( return Err((
StatusCode::FORBIDDEN, StatusCode::FORBIDDEN,
Json(json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } })), Json(
json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } }),
),
)); ));
} }
} }
// Verify host exists // Verify host exists
let host_exists: bool = sqlx::query_scalar( let host_exists: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM hosts WHERE id = $1)")
"SELECT EXISTS (SELECT 1 FROM hosts WHERE id = $1)",
)
.bind(host_id) .bind(host_id)
.fetch_one(&state.db) .fetch_one(&state.db)
.await .await
@ -191,10 +190,7 @@ async fn list_health_checks(
) )
})?; })?;
checks_with_results.push(HealthCheckWithResult { checks_with_results.push(HealthCheckWithResult { check, last_result });
check,
last_result,
});
} }
Ok(Json(HealthCheckListResponse { Ok(Json(HealthCheckListResponse {
@ -225,7 +221,9 @@ async fn create_health_check(
if !can_access { if !can_access {
return Err(( return Err((
StatusCode::FORBIDDEN, StatusCode::FORBIDDEN,
Json(json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } })), Json(
json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } }),
),
)); ));
} }
} }
@ -234,7 +232,9 @@ async fn create_health_check(
if req.check_type != "service" && req.check_type != "http" { if req.check_type != "service" && req.check_type != "http" {
return Err(( return Err((
StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST,
Json(json!({ "error": { "code": "invalid_check_type", "message": "check_type must be 'service' or 'http'" } })), Json(
json!({ "error": { "code": "invalid_check_type", "message": "check_type must be 'service' or 'http'" } }),
),
)); ));
} }
@ -242,13 +242,17 @@ async fn create_health_check(
if req.check_type == "service" && req.service_name.is_none() { if req.check_type == "service" && req.service_name.is_none() {
return Err(( return Err((
StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST,
Json(json!({ "error": { "code": "validation_error", "message": "service_name is required for service checks" } })), Json(
json!({ "error": { "code": "validation_error", "message": "service_name is required for service checks" } }),
),
)); ));
} }
if req.check_type == "http" && (req.url.is_none() || req.expected_body.is_none()) { if req.check_type == "http" && (req.url.is_none() || req.expected_body.is_none()) {
return Err(( return Err((
StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST,
Json(json!({ "error": { "code": "validation_error", "message": "url and expected_body are required for http checks" } })), Json(
json!({ "error": { "code": "validation_error", "message": "url and expected_body are required for http checks" } }),
),
)); ));
} }
@ -270,7 +274,9 @@ async fn create_health_check(
if count >= 5 { if count >= 5 {
return Err(( return Err((
StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST,
Json(json!({ "error": { "code": "limit_exceeded", "message": "Maximum 5 health checks per host" } })), Json(
json!({ "error": { "code": "limit_exceeded", "message": "Maximum 5 health checks per host" } }),
),
)); ));
} }
@ -288,7 +294,9 @@ async fn create_health_check(
tracing::error!(error = %e, "Failed to encrypt password"); tracing::error!(error = %e, "Failed to encrypt password");
( (
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({ "error": { "code": "internal_error", "message": "Encryption error" } })), Json(
json!({ "error": { "code": "internal_error", "message": "Encryption error" } }),
),
) )
})?; })?;
(Some(enc), Some(nonce)) (Some(enc), Some(nonce))
@ -342,7 +350,8 @@ async fn create_health_check(
}), }),
None, None,
None, None,
).await; )
.await;
Ok(( Ok((
StatusCode::CREATED, StatusCode::CREATED,
@ -377,7 +386,9 @@ async fn get_health_check(
if !can_access { if !can_access {
return Err(( return Err((
StatusCode::FORBIDDEN, StatusCode::FORBIDDEN,
Json(json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } })), Json(
json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } }),
),
)); ));
} }
} }
@ -429,10 +440,7 @@ async fn get_health_check(
) )
})?; })?;
Ok(Json(HealthCheckWithResult { Ok(Json(HealthCheckWithResult { check, last_result }))
check,
last_result,
}))
} }
// ── PUT /api/v1/hosts/{host_id}/health-checks/{check_id} ────────────────────── // ── PUT /api/v1/hosts/{host_id}/health-checks/{check_id} ──────────────────────
@ -457,7 +465,9 @@ async fn update_health_check(
if !can_access { if !can_access {
return Err(( return Err((
StatusCode::FORBIDDEN, StatusCode::FORBIDDEN,
Json(json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } })), Json(
json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } }),
),
)); ));
} }
} }
@ -499,7 +509,9 @@ async fn update_health_check(
tracing::error!(error = %e, "Failed to encrypt password"); tracing::error!(error = %e, "Failed to encrypt password");
( (
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({ "error": { "code": "internal_error", "message": "Encryption error" } })), Json(
json!({ "error": { "code": "internal_error", "message": "Encryption error" } }),
),
) )
})?; })?;
(Some(enc), Some(nonce)) (Some(enc), Some(nonce))
@ -549,7 +561,9 @@ async fn update_health_check(
if set_clauses.is_empty() { if set_clauses.is_empty() {
return Err(( return Err((
StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST,
Json(json!({ "error": { "code": "validation_error", "message": "No fields to update" } })), Json(
json!({ "error": { "code": "validation_error", "message": "No fields to update" } }),
),
)); ));
} }
@ -613,7 +627,8 @@ async fn update_health_check(
json!({ "check_id": check_id }), json!({ "check_id": check_id }),
None, None,
None, None,
).await; )
.await;
Ok(Json(json!({ "id": check_id, "updated": true }))) Ok(Json(json!({ "id": check_id, "updated": true })))
} }
@ -639,14 +654,14 @@ async fn delete_health_check(
if !can_access { if !can_access {
return Err(( return Err((
StatusCode::FORBIDDEN, StatusCode::FORBIDDEN,
Json(json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } })), Json(
json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } }),
),
)); ));
} }
} }
let deleted = sqlx::query( let deleted = sqlx::query("DELETE FROM host_health_checks WHERE id = $1 AND host_id = $2")
"DELETE FROM host_health_checks WHERE id = $1 AND host_id = $2",
)
.bind(check_id) .bind(check_id)
.bind(host_id) .bind(host_id)
.execute(&state.db) .execute(&state.db)
@ -677,7 +692,8 @@ async fn delete_health_check(
json!({ "check_id": check_id }), json!({ "check_id": check_id }),
None, None,
None, None,
).await; )
.await;
Ok(StatusCode::NO_CONTENT) Ok(StatusCode::NO_CONTENT)
} }
@ -703,7 +719,9 @@ async fn test_health_check(
if !can_access { if !can_access {
return Err(( return Err((
StatusCode::FORBIDDEN, StatusCode::FORBIDDEN,
Json(json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } })), Json(
json!({ "error": { "code": "forbidden", "message": "Not authorized for this host" } }),
),
)); ));
} }
} }
@ -794,7 +812,7 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
detail: "No service_name configured".to_string(), detail: "No service_name configured".to_string(),
latency_ms: None, latency_ms: None,
} }
} },
}; };
// Get host info for agent connection // Get host info for agent connection
@ -815,11 +833,14 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
detail: "Host not found".to_string(), detail: "Host not found".to_string(),
latency_ms: None, latency_ms: None,
} }
} },
}; };
// Build agent URL // Build agent URL
let agent_url = format!("https://{}:12443/api/v1/system/services/{}", ip, service_name); let agent_url = format!(
"https://{}:12443/api/v1/system/services/{}",
ip, service_name
);
let start = std::time::Instant::now(); let start = std::time::Instant::now();
@ -832,10 +853,15 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
detail: format!("Failed to build HTTP client: {}", e), detail: format!("Failed to build HTTP client: {}", e),
latency_ms: None, latency_ms: None,
} }
} },
}; };
match client.get(&agent_url).timeout(std::time::Duration::from_secs(10)).send().await { match client
.get(&agent_url)
.timeout(std::time::Duration::from_secs(10))
.send()
.await
{
Ok(resp) => { Ok(resp) => {
let latency = start.elapsed().as_millis() as i32; let latency = start.elapsed().as_millis() as i32;
let status = resp.status(); let status = resp.status();
@ -847,11 +873,14 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
// Try to parse as ApiResponse<ServiceStatusData> // Try to parse as ApiResponse<ServiceStatusData>
if let Ok(api_resp) = serde_json::from_str::<serde_json::Value>(&body) { if let Ok(api_resp) = serde_json::from_str::<serde_json::Value>(&body) {
if let Some(data) = api_resp.get("data") { if let Some(data) = api_resp.get("data") {
if let Some(healthy) = data.get("healthy").and_then(|v| v.as_bool()) { if let Some(healthy) = data.get("healthy").and_then(|v| v.as_bool())
let active_state = data.get("active_state") {
let active_state = data
.get("active_state")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.unwrap_or("unknown"); .unwrap_or("unknown");
let sub_state = data.get("sub_state") let sub_state = data
.get("sub_state")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.unwrap_or(""); .unwrap_or("");
return CheckResult { return CheckResult {
@ -867,7 +896,7 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
detail: format!("Failed to parse agent response"), detail: format!("Failed to parse agent response"),
latency_ms: Some(latency), latency_ms: Some(latency),
} }
} },
Err(e) => CheckResult { Err(e) => CheckResult {
healthy: false, healthy: false,
detail: format!("Failed to read response: {}", e), detail: format!("Failed to read response: {}", e),
@ -881,7 +910,7 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
latency_ms: Some(latency), latency_ms: Some(latency),
} }
} }
} },
Err(e) => { Err(e) => {
let latency = start.elapsed().as_millis() as i32; let latency = start.elapsed().as_millis() as i32;
if e.is_timeout() { if e.is_timeout() {
@ -897,7 +926,7 @@ async fn run_service_check(check: &HealthCheck, state: &AppState) -> CheckResult
latency_ms: Some(latency), latency_ms: Some(latency),
} }
} }
} },
} }
} }
@ -910,7 +939,7 @@ async fn run_http_check(check: &HealthCheck, state: &AppState) -> CheckResult {
detail: "No URL configured".to_string(), detail: "No URL configured".to_string(),
latency_ms: None, latency_ms: None,
} }
} },
}; };
let expected = match &check.expected_body { let expected = match &check.expected_body {
@ -921,7 +950,7 @@ async fn run_http_check(check: &HealthCheck, state: &AppState) -> CheckResult {
detail: "No expected_body configured".to_string(), detail: "No expected_body configured".to_string(),
latency_ms: None, latency_ms: None,
} }
} },
}; };
// Build HTTP client // Build HTTP client
@ -983,14 +1012,14 @@ async fn run_http_check(check: &HealthCheck, state: &AppState) -> CheckResult {
}, },
latency_ms: Some(latency), latency_ms: Some(latency),
} }
} },
Err(e) => CheckResult { Err(e) => CheckResult {
healthy: false, healthy: false,
detail: format!("Failed to read response: {}", e), detail: format!("Failed to read response: {}", e),
latency_ms: Some(latency), latency_ms: Some(latency),
}, },
} }
} },
Err(e) => { Err(e) => {
let latency = start.elapsed().as_millis() as i32; let latency = start.elapsed().as_millis() as i32;
if e.is_timeout() { if e.is_timeout() {
@ -1006,7 +1035,7 @@ async fn run_http_check(check: &HealthCheck, state: &AppState) -> CheckResult {
latency_ms: Some(latency), latency_ms: Some(latency),
} }
} }
} },
} }
} }
@ -1030,20 +1059,32 @@ fn build_agent_http_client(state: &AppState) -> Result<reqwest::Client, String>
); );
// Add CA cert (mandatory since we disabled built-in root certs) // Add CA cert (mandatory since we disabled built-in root certs)
let ca_pem = std::fs::read(ca_cert_path).map_err(|e| format!("Read CA cert {}: {}", ca_cert_path, e))?; let ca_pem =
std::fs::read(ca_cert_path).map_err(|e| format!("Read CA cert {}: {}", ca_cert_path, e))?;
tracing::info!(ca_pem_len = ca_pem.len(), "CA cert read"); tracing::info!(ca_pem_len = ca_pem.len(), "CA cert read");
let ca = reqwest::Certificate::from_pem(&ca_pem).map_err(|e| format!("Parse CA cert: {}", e))?; let ca =
reqwest::Certificate::from_pem(&ca_pem).map_err(|e| format!("Parse CA cert: {}", e))?;
builder = builder.add_root_certificate(ca); builder = builder.add_root_certificate(ca);
// Add client cert + key for mTLS // Add client cert + key for mTLS
let client_cert_exists = std::path::Path::new(client_cert_path).exists(); let client_cert_exists = std::path::Path::new(client_cert_path).exists();
let client_key_exists = std::path::Path::new(client_key_path).exists(); let client_key_exists = std::path::Path::new(client_key_path).exists();
tracing::info!(client_cert_exists, client_key_exists, "Checking client cert files"); tracing::info!(
client_cert_exists,
client_key_exists,
"Checking client cert files"
);
if client_cert_exists && client_key_exists { if client_cert_exists && client_key_exists {
let client_pem = std::fs::read(client_cert_path).map_err(|e| format!("Read client cert: {}", e))?; let client_pem =
let key_pem = std::fs::read(client_key_path).map_err(|e| format!("Read client key: {}", e))?; std::fs::read(client_cert_path).map_err(|e| format!("Read client cert: {}", e))?;
tracing::info!(cert_len = client_pem.len(), key_len = key_pem.len(), "Client cert/key read"); let key_pem =
std::fs::read(client_key_path).map_err(|e| format!("Read client key: {}", e))?;
tracing::info!(
cert_len = client_pem.len(),
key_len = key_pem.len(),
"Client cert/key read"
);
let mut combined = Vec::new(); let mut combined = Vec::new();
combined.extend_from_slice(&client_pem); combined.extend_from_slice(&client_pem);
combined.extend_from_slice(&key_pem); combined.extend_from_slice(&key_pem);
@ -1062,6 +1103,6 @@ fn build_agent_http_client(state: &AppState) -> Result<reqwest::Client, String>
Err(e) => { Err(e) => {
tracing::error!(error = %e, "Failed to build reqwest client"); tracing::error!(error = %e, "Failed to build reqwest client");
Err(format!("Build client: {}", e)) Err(format!("Build client: {}", e))
} },
} }
} }

View File

@ -31,7 +31,10 @@ pub fn router() -> Router<AppState> {
Router::new() Router::new()
.route("/", get(list_hosts).post(register_host)) .route("/", get(list_hosts).post(register_host))
.route("/{id}", get(get_host).delete(remove_host)) .route("/{id}", get(get_host).delete(remove_host))
.route("/{id}/groups", get(list_host_groups).post(add_host_to_group)) .route(
"/{id}/groups",
get(list_host_groups).post(add_host_to_group),
)
.route("/{id}/groups/{group_id}", delete(remove_host_from_group)) .route("/{id}/groups/{group_id}", delete(remove_host_from_group))
.route("/{id}/refresh", post(refresh_host)) .route("/{id}/refresh", post(refresh_host))
} }

View File

@ -4,6 +4,7 @@ pub mod azure_sso;
pub mod ca; pub mod ca;
pub mod discovery; pub mod discovery;
pub mod groups; pub mod groups;
pub mod health_checks;
pub mod hosts; pub mod hosts;
pub mod jobs; pub mod jobs;
pub mod maintenance_windows; pub mod maintenance_windows;
@ -11,6 +12,5 @@ pub mod settings;
pub mod status; pub mod status;
pub mod users; pub mod users;
pub mod ws; pub mod ws;
pub mod health_checks;
pub mod reports; pub mod reports;

View File

@ -263,34 +263,32 @@ async fn run_service_check(
let detail = if data.healthy { let detail = if data.healthy {
format!( format!(
"Service '{}' is {}/{} (enabled: {})", "Service '{}' is {}/{} (enabled: {})",
data.name, data.name, data.active_state, data.sub_state, data.enabled_state
data.active_state,
data.sub_state,
data.enabled_state
) )
} else { } else {
format!( format!(
"Service '{}' status: {}/{} (unhealthy, enabled: {})", "Service '{}' status: {}/{} (unhealthy, enabled: {})",
data.name, data.active_state, data.name, data.active_state, data.sub_state, data.enabled_state
data.sub_state,
data.enabled_state
) )
}; };
(data.healthy, detail) (data.healthy, detail)
}, },
Err(AgentClientError::Timeout) => { Err(AgentClientError::Timeout) => (
(false, format!("Agent timed out querying service '{service_name}'")) false,
}, format!("Agent timed out querying service '{service_name}'"),
Err(AgentClientError::Connect(_)) => { ),
(false, format!("Agent connection refused for service '{service_name}'")) Err(AgentClientError::Connect(_)) => (
}, false,
format!("Agent connection refused for service '{service_name}'"),
),
Err(AgentClientError::ApiError { code, message }) => { Err(AgentClientError::ApiError { code, message }) => {
// 404, 400, 500 etc. from the agent means the service is unhealthy. // 404, 400, 500 etc. from the agent means the service is unhealthy.
(false, format!("Agent error [{code}]: {message}")) (false, format!("Agent error [{code}]: {message}"))
}, },
Err(e) => { Err(e) => (
(false, format!("Agent error querying service '{service_name}': {e}")) false,
}, format!("Agent error querying service '{service_name}': {e}"),
),
} }
} }
@ -300,10 +298,7 @@ async fn run_service_check(
/// Execute an HTTP check by making a GET request to the configured URL. /// Execute an HTTP check by making a GET request to the configured URL.
/// Supports optional basic auth (decrypted from DB) and substring body matching. /// Supports optional basic auth (decrypted from DB) and substring body matching.
async fn run_http_check( async fn run_http_check(check: &HealthCheckRow, crypto_key: &[u8; 32]) -> (bool, String) {
check: &HealthCheckRow,
crypto_key: &[u8; 32],
) -> (bool, String) {
let url = match &check.url { let url = match &check.url {
Some(u) => u.clone(), Some(u) => u.clone(),
None => { None => {
@ -325,7 +320,9 @@ async fn run_http_check(
.build() .build()
.unwrap_or_else(|_| reqwest::Client::new()) .unwrap_or_else(|_| reqwest::Client::new())
} else { } else {
client_builder.build().unwrap_or_else(|_| reqwest::Client::new()) client_builder
.build()
.unwrap_or_else(|_| reqwest::Client::new())
}; };
// Build the request. // Build the request.
@ -334,21 +331,22 @@ async fn run_http_check(
// Add basic auth if configured. // Add basic auth if configured.
if let Some(user) = &check.basic_auth_user { if let Some(user) = &check.basic_auth_user {
// Decrypt the password if present. // Decrypt the password if present.
let password = match (&check.basic_auth_pass_encrypted, &check.basic_auth_pass_nonce) { let password = match (
(Some(enc), Some(nonce)) => { &check.basic_auth_pass_encrypted,
match crypto::decrypt(enc, nonce, crypto_key) { &check.basic_auth_pass_nonce,
) {
(Some(enc), Some(nonce)) => match crypto::decrypt(enc, nonce, crypto_key) {
Ok(p) => p, Ok(p) => p,
Err(e) => { Err(e) => {
return ( return (false, format!("Failed to decrypt basic auth password: {e}"));
false,
format!("Failed to decrypt basic auth password: {e}"),
);
}, },
}
}, },
_ => { _ => {
// No encrypted password stored — treat as missing credentials. // No encrypted password stored — treat as missing credentials.
return (false, "HTTP check has basic_auth_user but no encrypted password".to_string()); return (
false,
"HTTP check has basic_auth_user but no encrypted password".to_string(),
);
}, },
}; };
request = request.basic_auth(user.as_str(), Some(password.as_str())); request = request.basic_auth(user.as_str(), Some(password.as_str()));
@ -382,7 +380,10 @@ async fn run_http_check(
let body = match response.text().await { let body = match response.text().await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
return (false, format!("HTTP check failed to read response body: {e}")); return (
false,
format!("HTTP check failed to read response body: {e}"),
);
}, },
}; };
@ -391,14 +392,15 @@ async fn run_http_check(
if !body.contains(expected) { if !body.contains(expected) {
return ( return (
false, false,
format!( format!("HTTP check body mismatch for {url}: expected substring not found"),
"HTTP check body mismatch for {url}: expected substring not found"
),
); );
} }
} }
(true, format!("HTTP check OK for {url} (status {})", status.as_u16())) (
true,
format!("HTTP check OK for {url} (status {})", status.as_u16()),
)
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────

View File

@ -870,7 +870,11 @@ async fn sync_job_status(pool: &PgPool, job_id: Uuid) {
let new_status: &str; let new_status: &str;
let set_completed: bool; let set_completed: bool;
if counts.running_count > 0 || counts.pending_count > 0 || counts.queued_count > 0 || counts.waiting_health_check_count > 0 { if counts.running_count > 0
|| counts.pending_count > 0
|| counts.queued_count > 0
|| counts.waiting_health_check_count > 0
{
// Still work in flight — keep parent running. // Still work in flight — keep parent running.
new_status = "running"; new_status = "running";
set_completed = false; set_completed = false;
@ -1009,7 +1013,8 @@ pub async fn retry_pending_jobs(pool: PgPool, config: Arc<AppConfig>) {
WHERE pjh.status IN ('pending', 'waiting_health_check') WHERE pjh.status IN ('pending', 'waiting_health_check')
AND pjh.retry_next_at <= NOW() AND pjh.retry_next_at <= NOW()
AND j.status != 'cancelled' AND j.status != 'cancelled'
"#,) "#,
)
.fetch_all(&pool) .fetch_all(&pool)
.await .await
{ {

View File

@ -19,9 +19,9 @@ use tokio::sync::Mutex;
use tokio_tungstenite::{connect_async_tls_with_config, tungstenite::protocol::Message, Connector}; use tokio_tungstenite::{connect_async_tls_with_config, tungstenite::protocol::Message, Connector};
use uuid::Uuid; use uuid::Uuid;
use pm_agent_client::client::AgentClient;
use pm_agent_client::client::DEFAULT_AGENT_PORT; use pm_agent_client::client::DEFAULT_AGENT_PORT;
use pm_core::config::AppConfig; use pm_core::config::AppConfig;
use pm_agent_client::client::AgentClient;
// ── Types ───────────────────────────────────────────────────────────────────── // ── Types ─────────────────────────────────────────────────────────────────────
@ -254,7 +254,6 @@ async fn build_tls_config(config: &AppConfig) -> anyhow::Result<TlsClientConfig>
Ok(config) Ok(config)
} }
// ── Per-job relay ───────────────────────────────────────────────────────────── // ── Per-job relay ─────────────────────────────────────────────────────────────
async fn relay_one_job( async fn relay_one_job(