git-echo/linux_patch_manager
Private
Public Access
1
0
Fork 0
Code Releases 8 Wiki Activity

style: Apply rustfmt with stable-only config
Some checks failed
CI Pipeline / Clippy Lints (push) Failing after 0s
Details
CI Pipeline / Rust Unit Tests (push) Failing after 0s
Details
CI Pipeline / Rust Format Check (push) Successful in 4s
Details
CI Pipeline / Frontend Lint & Type Check (push) Failing after 0s
Details
CI Pipeline / Security Audit (push) Failing after 3s
Details
CI Pipeline / Build .deb & Release (push) Has been skipped
Details

Browse Source 
- Fixed rustfmt.toml to only use stable options (removed nightly-only)
- Applied cargo fmt --all to fix formatting violations
- Stable options: edition=2021, max_width=100, reorder_imports/modules, match_block_trailing_comma
This commit is contained in:
Echo
2026-04-24 15:32:50 +00:00
parent f0fe5f5fd1
commit 5a4d4d583e
44 changed files with 1498 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
crates/pm-worker/src/agent_loader.rs
View File

@ -34,12 +34,12 @@ pub fn load_agent_certs(security: &SecurityConfig) -> anyhow::Result<AgentCerts>
})?;
let ca_cert = std::fs::read(&security.ca_cert_path).map_err(|e| {
anyhow::anyhow!(
"Failed to read CA cert '{}': {}",
security.ca_cert_path,
e
)
anyhow::anyhow!("Failed to read CA cert '{}': {}", security.ca_cert_path, e)
})?;
Ok(AgentCerts { client_cert, client_key, ca_cert })
Ok(AgentCerts {
client_cert,
client_key,
ca_cert,
})
}

43
crates/pm-worker/src/email.rs
View File

@ -80,7 +80,8 @@ async fn load_notification_settings(pool: &PgPool) -> NotificationSettings {
.unwrap_or_default()
};
let recipients: Vec<String> = serde_json::from_str(&get("notification_email_recipients")).unwrap_or_default();
let recipients: Vec<String> =
serde_json::from_str(&get("notification_email_recipients")).unwrap_or_default();
NotificationSettings {
email_enabled: get("notification_email_enabled") == "true",
@ -90,9 +91,7 @@ async fn load_notification_settings(pool: &PgPool) -> NotificationSettings {
}
/// Build an async SMTP transport from settings.
fn build_transport(
settings: &SmtpSettings,
) -> Result<AsyncSmtpTransport<Tokio1Executor>, String> {
fn build_transport(settings: &SmtpSettings) -> Result<AsyncSmtpTransport<Tokio1Executor>, String> {
match settings.tls_mode.as_str() {
"tls" => {
let mut builder = AsyncSmtpTransport::<Tokio1Executor>::relay(&settings.host)
@ -105,7 +104,7 @@ fn build_transport(
));
}
Ok(builder.build())
}
},
"starttls" => {
let mut builder = AsyncSmtpTransport::<Tokio1Executor>::starttls_relay(&settings.host)
.map_err(|e| format!("STARTTLS relay error: {}", e))?;
@ -117,11 +116,12 @@ fn build_transport(
));
}
Ok(builder.build())
}
},
_ => {
// "none" — plaintext / no TLS
let mut builder = AsyncSmtpTransport::<Tokio1Executor>::builder_dangerous(&settings.host)
.port(settings.port);
let mut builder =
AsyncSmtpTransport::<Tokio1Executor>::builder_dangerous(&settings.host)
.port(settings.port);
if !settings.username.is_empty() {
builder = builder.credentials(Credentials::new(
settings.username.clone(),
@ -129,21 +129,17 @@ fn build_transport(
));
}
Ok(builder.build())
}
},
}
}
/// Send an email notification. Returns true if the email was sent successfully.
async fn send_email(
pool: &PgPool,
subject: &str,
body: &str,
) -> bool {
async fn send_email(pool: &PgPool, subject: &str, body: &str) -> bool {
let smtp = match load_smtp_settings(pool).await {
s if !s.enabled => {
tracing::debug!("SMTP not enabled, skipping email notification");
return false;
}
},
s => s,
};
@ -169,7 +165,7 @@ async fn send_email(
Err(e) => {
tracing::error!(error = %e, "Invalid from address for email notification");
return false;
}
},
};
let mut builder = Message::builder()
@ -184,7 +180,7 @@ async fn send_email(
Err(e) => {
tracing::error!(error = %e, recipient = %recipient, "Invalid recipient address");
continue;
}
},
};
builder = builder.to(mailbox);
}
@ -194,7 +190,7 @@ async fn send_email(
Err(e) => {
tracing::error!(error = %e, "Failed to build email message");
return false;
}
},
};
let transport = match build_transport(&smtp) {
@ -202,18 +198,18 @@ async fn send_email(
Err(e) => {
tracing::error!(error = %e, "Failed to build SMTP transport");
return false;
}
},
};
match transport.send(email).await {
Ok(_) => {
tracing::info!(subject, "Email notification sent successfully");
true
}
},
Err(e) => {
tracing::error!(error = %e, subject, "Failed to send email notification");
false
}
},
}
}
@ -300,7 +296,10 @@ pub async fn send_maintenance_window_reminder_email(
window_label: &str,
start_at: &str,
) {
let subject = format!("[Patch Manager] Upcoming Maintenance Window: {}", window_label);
let subject = format!(
"[Patch Manager] Upcoming Maintenance Window: {}",
window_label
);
let body = format!(
"Maintenance window reminder:\n\
Host: {host_fqdn}\n\

51
crates/pm-worker/src/health_poller.rs
View File

@ -7,15 +7,9 @@
use std::sync::Arc;
use pm_agent_client::{AgentClient, AgentClientError};
use pm_core::{
config::AppConfig,
models::HostHealthStatus,
};
use pm_core::{config::AppConfig, models::HostHealthStatus};
use sqlx::{FromRow, PgPool};
use tokio::{
sync::Semaphore,
time,
};
use tokio::{sync::Semaphore, time};
use uuid::Uuid;
use crate::agent_loader::load_agent_certs;
@ -37,10 +31,7 @@ pub async fn run_health_poller(pool: PgPool, config: Arc<AppConfig>) {
let interval_secs = config.worker.health_poll_interval_secs;
let mut ticker = time::interval(std::time::Duration::from_secs(interval_secs));
tracing::info!(
interval_secs,
"Health poller started"
);
tracing::info!(interval_secs, "Health poller started");
loop {
ticker.tick().await;
@ -51,7 +42,7 @@ pub async fn run_health_poller(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "Health poller: failed to load agent certs — skipping cycle");
continue;
}
},
};
let client_cert = Arc::new(certs.client_cert);
@ -69,7 +60,7 @@ pub async fn run_health_poller(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "Health poller: failed to fetch hosts");
continue;
}
},
};
if hosts.is_empty() {
@ -107,7 +98,7 @@ pub async fn run_health_poller(pool: PgPool, config: Arc<AppConfig>) {
Ok(HostHealthStatus::Healthy) => healthy += 1,
Ok(HostHealthStatus::Degraded) => degraded += 1,
Ok(HostHealthStatus::Unreachable) => unreachable += 1,
Ok(_) => {}
Ok(_) => {},
Err(e) => tracing::error!(error = %e, "Health poller task panicked"),
}
}
@ -144,25 +135,37 @@ async fn poll_host_health(
error = %e,
"Health poller: failed to build AgentClient"
);
(HostHealthStatus::Unreachable, serde_json::Value::Object(Default::default()))
}
(
HostHealthStatus::Unreachable,
serde_json::Value::Object(Default::default()),
)
},
Ok(client) => match client.health().await {
Ok(data) => {
let payload = serde_json::to_value(&data).unwrap_or_default();
(HostHealthStatus::Healthy, payload)
}
},
Err(AgentClientError::Timeout) => {
tracing::warn!(host_id = %host.id, "Health poller: agent timed out");
(HostHealthStatus::Unreachable, serde_json::Value::Object(Default::default()))
}
(
HostHealthStatus::Unreachable,
serde_json::Value::Object(Default::default()),
)
},
Err(AgentClientError::Connect(_)) => {
tracing::warn!(host_id = %host.id, "Health poller: agent connection refused");
(HostHealthStatus::Unreachable, serde_json::Value::Object(Default::default()))
}
(
HostHealthStatus::Unreachable,
serde_json::Value::Object(Default::default()),
)
},
Err(e) => {
tracing::warn!(host_id = %host.id, error = %e, "Health poller: agent error");
(HostHealthStatus::Degraded, serde_json::Value::Object(Default::default()))
}
(
HostHealthStatus::Degraded,
serde_json::Value::Object(Default::default()),
)
},
},
};

172
crates/pm-worker/src/job_executor.rs
View File

@ -15,7 +15,7 @@
use std::sync::Arc;
use chrono::{Duration as ChronoDuration, Utc};
use pm_agent_client::{AgentClient, types::ApplyPatchesRequest};
use pm_agent_client::{types::ApplyPatchesRequest, AgentClient};
use pm_core::config::AppConfig;
use sqlx::{FromRow, PgPool};
use tokio::{sync::Semaphore, time};
@ -71,13 +71,13 @@ struct RetryRow {
#[derive(Debug, FromRow)]
struct StatusCounts {
running_count: i64,
pending_count: i64,
queued_count: i64,
running_count: i64,
pending_count: i64,
queued_count: i64,
succeeded_count: i64,
failed_count: i64,
failed_count: i64,
cancelled_count: i64,
total_count: i64,
total_count: i64,
}
// ─────────────────────────────────────────────────────────────────────────────
@ -125,12 +125,8 @@ async fn run_notify_listener(pool: PgPool, config: Arc<AppConfig>) {
/// Inner NOTIFY loop — returns `Err` only on a fatal connection error so the
/// outer loop can reconnect.
async fn notify_listen_loop(
pool: &PgPool,
config: &Arc<AppConfig>,
) -> anyhow::Result<()> {
let mut listener =
sqlx::postgres::PgListener::connect(&config.database.url).await?;
async fn notify_listen_loop(pool: &PgPool, config: &Arc<AppConfig>) -> anyhow::Result<()> {
let mut listener = sqlx::postgres::PgListener::connect(&config.database.url).await?;
listener.listen("job_enqueued").await?;
tracing::debug!("Job executor NOTIFY listener connected");
@ -148,7 +144,7 @@ async fn notify_listen_loop(
"Job executor: invalid UUID in job_enqueued payload"
);
continue;
}
},
};
let (p, c) = (pool.clone(), config.clone());
@ -301,7 +297,7 @@ pub async fn process_job(pool: PgPool, config: Arc<AppConfig>, job_id: Uuid) {
Err(e) => {
tracing::error!(%job_id, error = %e, "process_job: failed to fetch queued hosts");
return;
}
},
};
if hosts.is_empty() {
@ -317,11 +313,11 @@ pub async fn process_job(pool: PgPool, config: Arc<AppConfig>, job_id: Uuid) {
Err(e) => {
tracing::error!(%job_id, error = %e, "process_job: semaphore closed");
break;
}
},
};
let (p, c) = (pool.clone(), config.clone());
let pjh_id = host.id;
let pjh_id = host.id;
let host_id = host.host_id;
tokio::spawn(async move {
@ -338,11 +334,11 @@ pub async fn process_job(pool: PgPool, config: Arc<AppConfig>, job_id: Uuid) {
/// Connect to a single host agent, submit the patch job, and record the
/// agent-assigned async job ID for later polling.
async fn execute_host_job(
pool: PgPool,
config: Arc<AppConfig>,
job_id: Uuid,
pool: PgPool,
config: Arc<AppConfig>,
job_id: Uuid,
host_id: Uuid,
pjh_id: Uuid,
pjh_id: Uuid,
) {
tracing::info!(%job_id, %host_id, %pjh_id, "execute_host_job: starting");
@ -364,34 +360,33 @@ async fn execute_host_job(
)
.await;
return;
}
},
Err(e) => {
tracing::error!(%host_id, error = %e, "execute_host_job: DB error fetching host");
handle_host_failure(pool, pjh_id, format!("DB error fetching host: {e}")).await;
return;
}
},
};
// ── 2. Fetch the job's patch_selection ──────────────────────────────────
let patch_sel: JobPatchSelection = match sqlx::query_as(
"SELECT patch_selection FROM patch_jobs WHERE id = $1",
)
.bind(job_id)
.fetch_optional(&pool)
.await
{
Ok(Some(row)) => row,
Ok(None) => {
tracing::error!(%job_id, "execute_host_job: parent job not found");
handle_host_failure(pool, pjh_id, format!("Parent job {job_id} not found")).await;
return;
}
Err(e) => {
tracing::error!(%job_id, error = %e, "execute_host_job: DB error fetching job");
handle_host_failure(pool, pjh_id, format!("DB error fetching job: {e}")).await;
return;
}
};
let patch_sel: JobPatchSelection =
match sqlx::query_as("SELECT patch_selection FROM patch_jobs WHERE id = $1")
.bind(job_id)
.fetch_optional(&pool)
.await
{
Ok(Some(row)) => row,
Ok(None) => {
tracing::error!(%job_id, "execute_host_job: parent job not found");
handle_host_failure(pool, pjh_id, format!("Parent job {job_id} not found")).await;
return;
},
Err(e) => {
tracing::error!(%job_id, error = %e, "execute_host_job: DB error fetching job");
handle_host_failure(pool, pjh_id, format!("DB error fetching job: {e}")).await;
return;
},
};
let packages: Vec<String> =
serde_json::from_value(patch_sel.patch_selection).unwrap_or_default();
@ -403,7 +398,7 @@ async fn execute_host_job(
tracing::error!(%host_id, error = %e, "execute_host_job: failed to load agent certs");
handle_host_failure(pool, pjh_id, format!("Failed to load agent certs: {e}")).await;
return;
}
},
};
// ── 4. Build AgentClient ─────────────────────────────────────────────────
@ -419,7 +414,7 @@ async fn execute_host_job(
tracing::error!(%host_id, error = %e, "execute_host_job: failed to build AgentClient");
handle_host_failure(pool, pjh_id, format!("Failed to build agent client: {e}")).await;
return;
}
},
};
// ── 5. Mark pjh as running ───────────────────────────────────────────────
@ -439,7 +434,10 @@ async fn execute_host_job(
}
// ── 6. Submit the patch job to the agent ─────────────────────────────────
let req = ApplyPatchesRequest { packages, allow_reboot: true };
let req = ApplyPatchesRequest {
packages,
allow_reboot: true,
};
match client.apply_patches(&req).await {
Ok(resp) => {
@ -450,13 +448,12 @@ async fn execute_host_job(
);
// ── 7. Store agent_job_id; status stays 'running' (agent is async) ──
if let Err(e) = sqlx::query(
"UPDATE patch_job_hosts SET agent_job_id = $1 WHERE id = $2",
)
.bind(&resp.job_id)
.bind(pjh_id)
.execute(&pool)
.await
if let Err(e) =
sqlx::query("UPDATE patch_job_hosts SET agent_job_id = $1 WHERE id = $2")
.bind(&resp.job_id)
.bind(pjh_id)
.execute(&pool)
.await
{
tracing::error!(
%pjh_id,
@ -464,11 +461,11 @@ async fn execute_host_job(
"execute_host_job: failed to store agent_job_id"
);
}
}
},
Err(e) => {
tracing::warn!(%pjh_id, error = %e, "execute_host_job: agent rejected job");
handle_host_failure(pool, pjh_id, format!("Agent error: {e}")).await;
}
},
}
}
@ -498,7 +495,7 @@ pub async fn poll_running_jobs(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "poll_running_jobs: DB query failed");
return;
}
},
};
for row in rows {
@ -510,11 +507,7 @@ pub async fn poll_running_jobs(pool: PgPool, config: Arc<AppConfig>) {
}
/// Poll one running host entry and update its status from the agent response.
async fn poll_single_host(
pool: PgPool,
config: Arc<AppConfig>,
row: PatchJobHostRunning,
) {
async fn poll_single_host(pool: PgPool, config: Arc<AppConfig>, row: PatchJobHostRunning) {
let certs = match load_agent_certs(&config.security) {
Ok(c) => c,
Err(e) => {
@ -524,7 +517,7 @@ async fn poll_single_host(
"poll_single_host: failed to load agent certs"
);
return;
}
},
};
let client = match AgentClient::new(
@ -542,7 +535,7 @@ async fn poll_single_host(
"poll_single_host: failed to build AgentClient"
);
return;
}
},
};
let status = match client.job_status(&row.agent_job_id).await {
@ -555,7 +548,7 @@ async fn poll_single_host(
"poll_single_host: agent status call failed"
);
return;
}
},
};
match status.status.as_str() {
@ -578,14 +571,14 @@ async fn poll_single_host(
tracing::error!(pjh_id = %row.id, error = %e, "poll_single_host: update failed");
}
sync_job_status(&pool, row.job_id).await;
}
},
"failed" => {
tracing::warn!(pjh_id = %row.id, "poll_single_host: agent job failed");
let err_msg = status
.error
.unwrap_or_else(|| "Agent reported failure (no detail)".to_string());
handle_host_failure(pool, row.id, err_msg).await;
}
},
"running" | "queued" => {
// Still in progress — nothing to update; will poll again next cycle.
tracing::debug!(
@ -593,14 +586,14 @@ async fn poll_single_host(
agent_status = %status.status,
"poll_single_host: job still in progress"
);
}
},
other => {
tracing::warn!(
pjh_id = %row.id,
agent_status = %other,
"poll_single_host: unexpected agent status — ignoring"
);
}
},
}
}
@ -624,7 +617,7 @@ async fn handle_host_failure(pool: PgPool, pjh_id: Uuid, error_msg: String) {
Err(e) => {
tracing::error!(%pjh_id, error = %e, "handle_host_failure: DB error fetching retry row");
return;
}
},
};
let row = match row {
@ -632,7 +625,7 @@ async fn handle_host_failure(pool: PgPool, pjh_id: Uuid, error_msg: String) {
None => {
tracing::error!(%pjh_id, "handle_host_failure: pjh row not found");
return;
}
},
};
if row.retry_count < 3 {
@ -736,7 +729,7 @@ async fn sync_job_status(pool: &PgPool, job_id: Uuid) {
Err(e) => {
tracing::error!(%job_id, error = %e, "sync_job_status: DB query failed");
return;
}
},
};
// Determine the aggregate status.
@ -745,19 +738,19 @@ async fn sync_job_status(pool: &PgPool, job_id: Uuid) {
if counts.running_count > 0 || counts.pending_count > 0 || counts.queued_count > 0 {
// Still work in flight — keep parent running.
new_status = "running";
new_status = "running";
set_completed = false;
} else if counts.total_count > 0 && counts.succeeded_count == counts.total_count {
// Every host succeeded.
new_status = "succeeded";
new_status = "succeeded";
set_completed = true;
} else if counts.total_count > 0 && counts.cancelled_count == counts.total_count {
// Every host cancelled.
new_status = "cancelled";
new_status = "cancelled";
set_completed = true;
} else if counts.failed_count > 0 {
// At least one failure and nothing still active → failed (partial counts too).
new_status = "failed";
new_status = "failed";
set_completed = true;
} else {
// Fallback: nothing actionable yet.
@ -789,13 +782,11 @@ async fn sync_job_status(pool: &PgPool, job_id: Uuid) {
.execute(pool)
.await
} else {
sqlx::query(
"UPDATE patch_jobs SET status = $2 WHERE id = $1",
)
.bind(job_id)
.bind(new_status)
.execute(pool)
.await
sqlx::query("UPDATE patch_jobs SET status = $2 WHERE id = $1")
.bind(job_id)
.bind(new_status)
.execute(pool)
.await
};
if let Err(e) = result {
@ -812,13 +803,8 @@ async fn sync_job_status(pool: &PgPool, job_id: Uuid) {
let failed = counts.failed_count;
tokio::spawn(async move {
email::send_job_completion_email(
&pool_clone,
&job_id_str,
total,
succeeded,
failed,
).await;
email::send_job_completion_email(&pool_clone, &job_id_str, total, succeeded, failed)
.await;
// If there are failures, also send failure emails per host
if failed > 0 {
@ -838,16 +824,12 @@ async fn sync_job_status(pool: &PgPool, job_id: Uuid) {
Err(e) => {
tracing::error!(%job_id, error = %e, "sync_job_status: failed to fetch failed hosts for email");
Vec::new()
}
},
};
for (fqdn, error_msg) in failed_hosts {
email::send_patch_failure_email(
&pool_clone,
&fqdn,
&job_id_str,
&error_msg,
).await;
email::send_patch_failure_email(&pool_clone, &fqdn, &job_id_str, &error_msg)
.await;
}
}
});
@ -878,7 +860,7 @@ pub async fn retry_pending_jobs(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "retry_pending_jobs: DB query failed");
return;
}
},
};
for row in rows {

30
crates/pm-worker/src/main.rs
View File

@ -7,27 +7,23 @@ mod agent_loader;
mod audit_verifier;
mod email;
mod health_poller;
mod job_executor;
mod maintenance_scheduler;
mod patch_poller;
mod refresh_listener;
mod job_executor;
mod ws_relay;
use pm_core::{
config::AppConfig,
db,
logging,
};
use pm_core::{config::AppConfig, db, logging};
use sqlx::PgPool;
use std::{sync::Arc, time::Duration};
use tokio::time;
use audit_verifier::run_audit_verifier;
use health_poller::run_health_poller;
use job_executor::run_job_executor;
use maintenance_scheduler::run_maintenance_scheduler;
use patch_poller::run_patch_poller;
use refresh_listener::run_refresh_listener;
use job_executor::run_job_executor;
use ws_relay::run_ws_relay;
/// Minimum number of applied migrations the worker requires before
@ -44,16 +40,18 @@ async fn main() -> anyhow::Result<()> {
let config_path = std::env::var("PATCH_MANAGER_CONFIG")
.unwrap_or_else(|_| "/etc/patch-manager/config.toml".to_string());
let config = AppConfig::load(&config_path)
.unwrap_or_else(|_| {
eprintln!("Config file not found or invalid, using defaults");
AppConfig::default()
});
let config = AppConfig::load(&config_path).unwrap_or_else(|_| {
eprintln!("Config file not found or invalid, using defaults");
AppConfig::default()
});
// Initialize logging
logging::init(&config.logging);
tracing::info!(version = env!("CARGO_PKG_VERSION"), "patch-manager-worker starting");
tracing::info!(
version = env!("CARGO_PKG_VERSION"),
"patch-manager-worker starting"
);
// Initialize database pool
let pool = db::init_pool(&config.database).await?;
@ -114,17 +112,17 @@ async fn wait_for_schema(pool: &PgPool) -> anyhow::Result<()> {
Ok(count) if count >= REQUIRED_MIGRATION_COUNT => {
tracing::info!(migration_count = count, "Schema version check passed");
return Ok(());
}
},
Ok(count) => {
tracing::warn!(
migration_count = count,
required = REQUIRED_MIGRATION_COUNT,
"Schema not ready, waiting..."
);
}
},
Err(e) => {
tracing::warn!(error = %e, "Schema version check failed, retrying...");
}
},
}
if tokio::time::Instant::now() >= deadline {

2
crates/pm-worker/src/maintenance_scheduler.rs
View File

@ -144,7 +144,7 @@ async fn dispatch_open_window_jobs(pool: PgPool, config: Arc<AppConfig>) {
"dispatch_open_window_jobs: queued jobs query failed"
);
continue;
}
},
};
for job in job_ids {

39
crates/pm-worker/src/patch_poller.rs
View File

@ -9,10 +9,7 @@ use std::sync::Arc;
use pm_agent_client::AgentClient;
use pm_core::config::AppConfig;
use sqlx::{FromRow, PgPool};
use tokio::{
sync::Semaphore,
time,
};
use tokio::{sync::Semaphore, time};
use uuid::Uuid;
use crate::agent_loader::load_agent_certs;
@ -34,10 +31,7 @@ pub async fn run_patch_poller(pool: PgPool, config: Arc<AppConfig>) {
let interval_secs = config.worker.patch_poll_interval_secs;
let mut ticker = time::interval(std::time::Duration::from_secs(interval_secs));
tracing::info!(
interval_secs,
"Patch poller started"
);
tracing::info!(interval_secs, "Patch poller started");
loop {
ticker.tick().await;
@ -47,7 +41,7 @@ pub async fn run_patch_poller(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "Patch poller: failed to load agent certs — skipping cycle");
continue;
}
},
};
let client_cert = Arc::new(certs.client_cert);
@ -64,7 +58,7 @@ pub async fn run_patch_poller(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "Patch poller: failed to fetch hosts");
continue;
}
},
};
if hosts.is_empty() {
@ -102,16 +96,11 @@ pub async fn run_patch_poller(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "Patch poller task panicked");
failed += 1;
}
},
}
}
tracing::info!(
total,
succeeded,
failed,
"Patch poll cycle complete"
);
tracing::info!(total, succeeded, failed, "Patch poll cycle complete");
}
}
@ -135,7 +124,7 @@ async fn poll_host_patches(
Err(e) => {
tracing::warn!(host_id = %host.id, error = %e, "Patch poller: failed to build AgentClient");
return false;
}
},
};
// Fetch patches and packages concurrently.
@ -147,7 +136,7 @@ async fn poll_host_patches(
Err(e) => {
tracing::warn!(host_id = %host.id, error = %e, "Patch poller: patches() failed");
return false;
}
},
};
let packages_data = match packages_result {
@ -155,7 +144,7 @@ async fn poll_host_patches(
Err(e) => {
tracing::warn!(host_id = %host.id, error = %e, "Patch poller: packages_upgradable() failed");
return false;
}
},
};
let available_patches = serde_json::to_value(&patches_data.patches).unwrap_or_default();
@ -188,12 +177,10 @@ async fn poll_host_patches(
}
// Update hosts.last_patch_at.
if let Err(e) = sqlx::query(
"UPDATE hosts SET last_patch_at = NOW() WHERE id = $1",
)
.bind(host.id)
.execute(&pool)
.await
if let Err(e) = sqlx::query("UPDATE hosts SET last_patch_at = NOW() WHERE id = $1")
.bind(host.id)
.execute(&pool)
.await
{
tracing::error!(host_id = %host.id, error = %e, "Patch poller: failed to update last_patch_at");
}

62
crates/pm-worker/src/refresh_listener.rs
View File

@ -8,10 +8,7 @@
use std::sync::Arc;
use pm_agent_client::{AgentClient, AgentClientError};
use pm_core::{
config::AppConfig,
models::HostHealthStatus,
};
use pm_core::{config::AppConfig, models::HostHealthStatus};
use sqlx::{FromRow, PgPool};
use tokio::time;
use uuid::Uuid;
@ -46,8 +43,7 @@ pub async fn run_refresh_listener(pool: PgPool, config: Arc<AppConfig>) {
/// Inner loop — returns `Err` only on a fatal listener error so the outer
/// loop can reconnect.
async fn listen_loop(pool: &PgPool, config: &AppConfig) -> anyhow::Result<()> {
let mut listener =
sqlx::postgres::PgListener::connect(&config.database.url).await?;
let mut listener = sqlx::postgres::PgListener::connect(&config.database.url).await?;
listener.listen("refresh_requested").await?;
@ -68,7 +64,7 @@ async fn listen_loop(pool: &PgPool, config: &AppConfig) -> anyhow::Result<()> {
"Refresh listener: invalid UUID in notification payload"
);
continue;
}
},
};
// Fetch the host from the database.
@ -85,7 +81,7 @@ async fn listen_loop(pool: &PgPool, config: &AppConfig) -> anyhow::Result<()> {
None => {
tracing::warn!(%host_id, "Refresh listener: host not found");
continue;
}
},
};
// Load certs for this refresh.
@ -98,7 +94,7 @@ async fn listen_loop(pool: &PgPool, config: &AppConfig) -> anyhow::Result<()> {
"Refresh listener: failed to load agent certs"
);
continue;
}
},
};
// Spawn the actual work so the listener loop is not blocked.
@ -137,7 +133,7 @@ async fn refresh_host(
);
persist_health_unreachable(&pool, host.id).await;
return;
}
},
};
// ── Health ────────────────────────────────────────────────────────────
@ -145,15 +141,21 @@ async fn refresh_host(
Ok(data) => {
let payload = serde_json::to_value(&data).unwrap_or_default();
(HostHealthStatus::Healthy, payload)
}
},
Err(AgentClientError::Timeout) | Err(AgentClientError::Connect(_)) => {
tracing::warn!(host_id = %host.id, "Refresh: agent unreachable");
(HostHealthStatus::Unreachable, serde_json::Value::Object(Default::default()))
}
(
HostHealthStatus::Unreachable,
serde_json::Value::Object(Default::default()),
)
},
Err(e) => {
tracing::warn!(host_id = %host.id, error = %e, "Refresh: health error");
(HostHealthStatus::Degraded, serde_json::Value::Object(Default::default()))
}
(
HostHealthStatus::Degraded,
serde_json::Value::Object(Default::default()),
)
},
};
persist_health(&pool, host.id, &health_status, &health_payload).await;
@ -164,8 +166,7 @@ async fn refresh_host(
match (patches_result, packages_result) {
(Ok(patches_data), Ok(packages_data)) => {
let available_patches =
serde_json::to_value(&patches_data.patches).unwrap_or_default();
let available_patches = serde_json::to_value(&patches_data.patches).unwrap_or_default();
let installed_packages =
serde_json::to_value(&packages_data.packages).unwrap_or_default();
let patch_count = patches_data.total as i32;
@ -196,12 +197,10 @@ async fn refresh_host(
"Refresh: failed to insert patch data"
);
} else {
let _ = sqlx::query(
"UPDATE hosts SET last_patch_at = NOW() WHERE id = $1",
)
.bind(host.id)
.execute(&pool)
.await;
let _ = sqlx::query("UPDATE hosts SET last_patch_at = NOW() WHERE id = $1")
.bind(host.id)
.execute(&pool)
.await;
tracing::info!(
host_id = %host.id,
@ -210,14 +209,14 @@ async fn refresh_host(
"On-demand refresh complete"
);
}
}
},
(Err(e), _) | (_, Err(e)) => {
tracing::warn!(
host_id = %host.id,
error = %e,
"Refresh: failed to collect patch data"
);
}
},
}
}
@ -252,13 +251,12 @@ async fn persist_health(
);
}
if let Err(e) = sqlx::query(
"UPDATE hosts SET health_status = $2, last_health_at = NOW() WHERE id = $1",
)
.bind(host_id)
.bind(status)
.execute(pool)
.await
if let Err(e) =
sqlx::query("UPDATE hosts SET health_status = $2, last_health_at = NOW() WHERE id = $1")
.bind(host_id)
.bind(status)
.execute(pool)
.await
{
tracing::error!(%host_id, error = %e, "Refresh: failed to update host health_status");
}

86
crates/pm-worker/src/ws_relay.rs
View File

@ -5,27 +5,18 @@
//! DB row, and fire `pg_notify('job_update', payload_json)` so the browser WS
//! handler can forward the event to connected clients.
use std::{
collections::HashSet,
sync::Arc,
time::Duration,
};
use std::{collections::HashSet, sync::Arc, time::Duration};
use anyhow::Context;
use futures::StreamExt;
use rustls::{
pki_types::{CertificateDer, PrivateKeyDer},
ClientConfig as TlsClientConfig,
RootCertStore,
ClientConfig as TlsClientConfig, RootCertStore,
};
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
use tokio::sync::Mutex;
use tokio_tungstenite::{
connect_async_tls_with_config,
tungstenite::protocol::Message,
Connector,
};
use tokio_tungstenite::{connect_async_tls_with_config, tungstenite::protocol::Message, Connector};
use uuid::Uuid;
use pm_agent_client::client::DEFAULT_AGENT_PORT;
@ -84,7 +75,7 @@ pub async fn run_ws_relay(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "ws_relay: DB poll failed");
continue;
}
},
};
for row in rows {
@ -101,12 +92,12 @@ pub async fn run_ws_relay(pool: PgPool, config: Arc<AppConfig>) {
Err(e) => {
tracing::error!(error = %e, "ws_relay: TLS config error");
continue;
}
},
};
active.lock().await.insert(key);
let pool_c = pool.clone();
let pool_c = pool.clone();
let active_c = active.clone();
tokio::spawn(async move {
@ -164,12 +155,15 @@ async fn query_running_jobs(pool: &PgPool) -> anyhow::Result<Vec<RunningHostJob>
async fn build_tls_config(config: &AppConfig) -> anyhow::Result<TlsClientConfig> {
let sec = &config.security;
let cert_pem = tokio::fs::read(&sec.agent_client_cert_path).await
let cert_pem = tokio::fs::read(&sec.agent_client_cert_path)
.await
.with_context(|| format!("read agent client cert '{}'", sec.agent_client_cert_path))?;
let key_pem = tokio::fs::read(&sec.agent_client_key_path).await
.with_context(|| format!("read agent client key '{}'" , sec.agent_client_key_path))?;
let ca_pem = tokio::fs::read(&sec.ca_cert_path).await
.with_context(|| format!("read CA cert '{}'", sec.ca_cert_path))?;
let key_pem = tokio::fs::read(&sec.agent_client_key_path)
.await
.with_context(|| format!("read agent client key '{}'", sec.agent_client_key_path))?;
let ca_pem = tokio::fs::read(&sec.ca_cert_path)
.await
.with_context(|| format!("read CA cert '{}'", sec.ca_cert_path))?;
// Parse client certificate chain.
let client_certs: Vec<CertificateDer<'static>> = {
@ -207,8 +201,8 @@ async fn build_tls_config(config: &AppConfig) -> anyhow::Result<TlsClientConfig>
// ── Per-job relay ─────────────────────────────────────────────────────────────
async fn relay_one_job(
pool: &PgPool,
row: &RunningHostJob,
pool: &PgPool,
row: &RunningHostJob,
tls_config: Arc<TlsClientConfig>,
) -> anyhow::Result<()> {
let url = format!(
@ -229,7 +223,7 @@ async fn relay_one_job(
while let Some(frame) = stream.next().await {
let frame = match frame {
Ok(f) => f,
Ok(f) => f,
Err(e) => {
tracing::warn!(
error = %e,
@ -238,16 +232,16 @@ async fn relay_one_job(
"WS relay: stream error"
);
break;
}
},
};
let text = match frame {
Message::Text(t) => t.to_string(),
Message::Text(t) => t.to_string(),
Message::Binary(b) => String::from_utf8(b.into()).unwrap_or_default(),
Message::Close(_) => {
Message::Close(_) => {
tracing::info!(job_id = %row.job_id, "Agent WS closed cleanly");
break;
}
},
_ => continue,
};
@ -256,14 +250,14 @@ async fn relay_one_job(
}
let event: AgentWsEvent = match serde_json::from_str(&text) {
Ok(e) => e,
Ok(e) => e,
Err(e) => {
tracing::warn!(
error = %e, raw = %text,
"WS relay: unparseable agent frame"
);
continue;
}
},
};
process_event(pool, row, &event).await;
@ -287,17 +281,17 @@ async fn relay_one_job(
async fn process_event(pool: &PgPool, row: &RunningHostJob, event: &AgentWsEvent) {
// Map agent status string to DB job_status enum value.
let db_status = match event.status.as_str() {
"running" => "running",
"running" => "running",
"succeeded" => "succeeded",
"failed" => "failed",
"failed" => "failed",
"cancelled" => "cancelled",
other => {
tracing::warn!(status = %other, "WS relay: unknown agent status");
return;
}
},
};
let output = event.output.as_deref().unwrap_or("");
let output = event.output.as_deref().unwrap_or("");
let error_msg = event.error.as_deref();
// Determine timestamps based on terminal state.
@ -359,20 +353,20 @@ async fn process_event(pool: &PgPool, row: &RunningHostJob, event: &AgentWsEvent
// Fire pg_notify so browser WS handlers forward the event.
let payload = NotifyPayload {
job_id: row.job_id.to_string(),
host_id: row.host_id.to_string(),
status: db_status.to_string(),
output: event.output.clone(),
job_id: row.job_id.to_string(),
host_id: row.host_id.to_string(),
status: db_status.to_string(),
output: event.output.clone(),
error_message: event.error.clone(),
agent_job_id: row.agent_job_id.clone(),
};
let payload_json = match serde_json::to_string(&payload) {
Ok(s) => s,
Ok(s) => s,
Err(e) => {
tracing::error!(error = %e, "WS relay: failed to serialize notify payload");
return;
}
},
};
if let Err(e) = sqlx::query("SELECT pg_notify('job_update', $1)")
@ -418,11 +412,11 @@ async fn update_parent_job_status(pool: &PgPool, job_id: Uuid) {
.fetch_one(pool)
.await
{
Ok(n) => n,
Ok(n) => n,
Err(e) => {
tracing::error!(error = %e, %job_id, "update_parent_job_status: count query failed");
return;
}
},
};
if pending > 0 {
@ -437,14 +431,18 @@ async fn update_parent_job_status(pool: &PgPool, job_id: Uuid) {
.fetch_one(pool)
.await
{
Ok(n) => n,
Ok(n) => n,
Err(e) => {
tracing::error!(error = %e, %job_id, "update_parent_job_status: failed-count query failed");
return;
}
},
};
let final_status = if failed_count > 0 { "failed" } else { "succeeded" };
let final_status = if failed_count > 0 {
"failed"
} else {
"succeeded"
};
if let Err(e) = sqlx::query(
"UPDATE patch_jobs SET status = $1::job_status, completed_at = NOW() WHERE id = $2",