Database log is attached (partial)

2025-08-22 10:08:08 UTC:10.8.3.111(39468):water_admin@prefect:[25883]:STATEMENT: INSERT INTO task_run (flow_run_id, task_key, dynamic_key, flow_run_run_count, empirical_policy, task_inputs, tags, labels, name, run_count, expected_start_time, total_run_time, id, created, updated) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::INTEGER, $5, $6, $7::JSONB, $8::JSONB, $9::VARCHAR, $10::INTEGER, $11::TIMESTAMP WITH TIME ZONE, $12::INTERVAL, $13::UUID, $14::TIMESTAMP WITH TIME ZONE, $15::TIMESTAMP WITH TIME ZONE) ON CONFLICT (id) DO UPDATE SET flow_run_id = $16::UUID, task_key = $17::VARCHAR, dynamic_key = $18::VARCHAR, flow_run_run_count = $19::INTEGER, empirical_policy = $20, task_inputs = $21, tags = $22::JSONB, labels = $23::JSONB, name = $24::VARCHAR, run_count = $25::INTEGER, expected_start_time = $26::TIMESTAMP WITH TIME ZONE, total_run_time = $27::INTERVAL, id = $28::UUID, updated = $29::TIMESTAMP WITH TIME ZONE WHERE task_run.state_timestamp < $30::TIMESTAMP WITH TIME ZONE 2025-08-22 10:08:10 UTC:10.8.3.111(42686):water_admin@prefect:[27506]:ERROR: deadlock detected 2025-08-22 10:08:10 UTC:10.8.3.111(42686):water_admin@prefect:[27506]:DETAIL: Process 27506 waits for ShareLock on transaction 478591107; blocked by process 27353. Process 27353 waits for ShareLock on speculative token 17 of transaction 478591108; blocked by process 27506. Process 27506: INSERT INTO task_run (flow_run_id, task_key, dynamic_key, flow_run_run_count, empirical_policy, task_inputs, tags, labels, name, run_count, expected_start_time, start_time, total_run_time, id, created, updated) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::INTEGER, $5, $6, $7::JSONB, $8::JSONB, $9::VARCHAR, $10::INTEGER, $11::TIMESTAMP WITH TIME ZONE, $12::TIMESTAMP WITH TIME ZONE, $13::INTERVAL, $14::UUID, $15::TIMESTAMP WITH TIME ZONE, $16::TIMESTAMP WITH TIME ZONE) ON CONFLICT (id) DO UPDATE SET flow_run_id = $17::UUID, task_key = $18::VARCHAR, dynamic_key = $19::VARCHAR, flow_run_run_count = $20::INTEGER, empirical_policy = $21, task_inputs = $22, tags = $23::JSONB, labels = $24::JSONB, name = $25::VARCHAR, run_count = $26::INTEGER, expected_start_time = $27::TIMESTAMP WITH TIME ZONE, start_time = $28::TIMESTAMP WITH TIME ZONE, total_run_time = $29::INTERVAL, id = $30::UUID, updated = $31::TIMESTAMP WITH TIME ZONE WHERE task_run.state_timestamp < $32::TIMESTAMP WITH TIME ZONE Process 27353: UPDATE task_run SET state_id=$1::UUID, state_type=$2::state_type, state_name=$3::VARCHAR, state_timestamp=$4::TIMESTAMP WITH TIME ZONE, updated=now() WHERE task_run.id = $5::UUID AND (task_run.state_timestamp IS NULL OR task_run.state_timestamp < $6::TIMESTAMP WITH TIME ZONE) 2025-08-22 10:08:10 UTC:10.8.3.111(42686):water_admin@prefect:[27506]:HINT: See server log for query details. 2025-08-22 10:08:10 UTC:10.8.3.111(42686):water_admin@prefect:[27506]:CONTEXT: while inserting index tuple (9339915,21) in relation "uq_task_run__flow_run_id_task_key_dynamic_key" 2025-08-22 10:08:10 UTC:10.8.3.111(42686):water_admin@prefect:[27506]:STATEMENT: INSERT INTO task_run (flow_run_id, task_key, dynamic_key, flow_run_run_count, empirical_policy, task_inputs, tags, labels, name, run_count, expected_start_time, start_time, total_run_time, id, created, updated) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::INTEGER, $5, $6, $7::JSONB, $8::JSONB, $9::VARCHAR, $10::INTEGER, $11::TIMESTAMP WITH TIME ZONE, $12::TIMESTAMP WITH TIME ZONE, $13::INTERVAL, $14::UUID, $15::TIMESTAMP WITH TIME ZONE, $16::TIMESTAMP WITH TIME ZONE) ON CONFLICT (id) DO UPDATE SET flow_run_id = $17::UUID, task_key = $18::VARCHAR, dynamic_key = $19::VARCHAR, flow_run_run_count = $20::INTEGER, empirical_policy = $21, task_inputs = $22, tags = $23::JSONB, labels = $24::JSONB, name = $25::VARCHAR, run_count = $26::INTEGER, expected_start_time = $27::TIMESTAMP WITH TIME ZONE, start_time = $28::TIMESTAMP WITH TIME ZONE, total_run_time = $29::INTERVAL, id = $30::UUID, updated = $31::TIMESTAMP WITH TIME ZONE WHERE task_run.state_timestamp < $32::TIMESTAMP WITH TIME ZONE

Slack Conversation