diff --git a/qmanager/policies/base/queue_policy_base.hpp b/qmanager/policies/base/queue_policy_base.hpp index 6fa2e44df..cba124cb2 100644 --- a/qmanager/policies/base/queue_policy_base.hpp +++ b/qmanager/policies/base/queue_policy_base.hpp @@ -646,14 +646,28 @@ class queue_policy_base_t : public resource_model::queue_adapter_base_t { case job_state_kind_t::ALLOC_RUNNING: // deliberately fall through case job_state_kind_t::RUNNING: - if (cancel (h, job_it->second->id, R, true, full_removal) != 0) { - flux_log_error (flux_h, - "%s: .free RPC partial cancel failed for jobid " - "%jd", - __FUNCTION__, - static_cast (id)); - errno = EINVAL; - goto out; + if (!final) { + if (cancel (h, job_it->second->id, R, true, full_removal) != 0) { + flux_log_error (flux_h, + "%s: .free RPC partial cancel failed for jobid " + "%jd", + __FUNCTION__, + static_cast (id)); + errno = EINVAL; + goto out; + } + } else { + // Run a full cancel to clean up all remaining allocated resources + if (cancel (h, job_it->second->id, true) != 0) { + flux_log_error (flux_h, + "%s: .free RPC full cancel failed for jobid " + "%jd", + __FUNCTION__, + static_cast (id)); + errno = EPROTO; + goto out; + } + full_removal = true; } // We still want to run the sched loop even if there's an inconsistent state set_schedulability (true); @@ -664,24 +678,17 @@ class queue_policy_base_t : public resource_model::queue_adapter_base_t { job_it->second->state = job_state_kind_t::COMPLETE; // hold a reference to the shared_ptr to keep it alive // during cancel - auto job_sp = job_it->second; m_jobs.erase (job_it); - if (final && !full_removal) { - // This error condition indicates a discrepancy between core and sched. + if (full_removal && !final) { + // This error condition can indicate a discrepancy between core and sched, + // specifically that a partial cancel removed an allocation prior to + // receiving the final .free RPC from core. flux_log_error (flux_h, - "%s: Final .free RPC failed to remove all resources for " + "%s: removed allocation before final .free RPC for " "jobid " "%jd", __FUNCTION__, static_cast (id)); - // Run a full cancel to clean up all remaining allocated resources - if (cancel (h, job_sp->id, true) != 0) { - flux_log_error (flux_h, - "%s: .free RPC full cancel failed for jobid " - "%jd", - __FUNCTION__, - static_cast (id)); - } errno = EPROTO; goto out; } diff --git a/resource/planner/c/planner_multi_c_interface.cpp b/resource/planner/c/planner_multi_c_interface.cpp index 97c6f8104..c316429c7 100644 --- a/resource/planner/c/planner_multi_c_interface.cpp +++ b/resource/planner/c/planner_multi_c_interface.cpp @@ -423,8 +423,13 @@ extern "C" int planner_multi_rem_span (planner_multi_t *ctx, int64_t span_id) goto done; } for (i = 0; i < it->second.size (); ++i) { - if (planner_rem_span (ctx->plan_multi->get_planner_at (i), it->second[i]) == -1) - goto done; + // If executed after partial cancel, depending on pruning filter settings + // some spans may no longer exist. In that case the span_lookup value for + // the resource type will be -1. + if (it->second[i] != -1) { + if (planner_rem_span (ctx->plan_multi->get_planner_at (i), it->second[i]) == -1) + goto done; + } } ctx->plan_multi->get_span_lookup ().erase (it); rc = 0; diff --git a/resource/traversers/dfu_impl_update.cpp b/resource/traversers/dfu_impl_update.cpp index f97e95ac6..8ee0b4ba6 100644 --- a/resource/traversers/dfu_impl_update.cpp +++ b/resource/traversers/dfu_impl_update.cpp @@ -535,6 +535,12 @@ int dfu_impl_t::mod_plan (vtx_t u, int64_t jobid, modify_data_t &mod_data) span = alloc_span->second; if (mod_data.mod_type != job_modify_t::PARTIAL_CANCEL) { (*m_graph)[u].schedule.allocations.erase (alloc_span); + } else { + // This condition is encountered when the vertex is + // not associated with a broker rank. We may need + // extra logic here to handle more advanced partial + // cancel in the future. + goto done; } } else if ((res_span = (*m_graph)[u].schedule.reservations.find (jobid)) != (*m_graph)[u].schedule.reservations.end ()) { diff --git a/t/CMakeLists.txt b/t/CMakeLists.txt index 30d3a5369..82368ac13 100644 --- a/t/CMakeLists.txt +++ b/t/CMakeLists.txt @@ -29,6 +29,7 @@ set(ALL_TESTS t1024-alloc-check.t t1025-rv1-reload.t t1026-rv1-partial-release.t + t1027-rv1-partial-release-brokerless-resources.t t3000-jobspec.t t3001-resource-basic.t t3002-resource-prefix.t diff --git a/t/data/resource/jgfs/issue1284.json b/t/data/resource/jgfs/issue1284.json new file mode 100644 index 000000000..5ffd02d99 --- /dev/null +++ b/t/data/resource/jgfs/issue1284.json @@ -0,0 +1,2333 @@ +{ + "version": 1, + "execution": { + "R_lite": [ + { + "rank": "0", + "children": { + "core": "0-9" + } + } + ], + "starttime": 0.0, + "expiration": 0.0, + "nodelist": [ + "compute-01" + ] + }, + "scheduling": { + "graph": { + "directed": true, + "nodes": [ + { + "id": "0", + "metadata": { + "type": "cluster", + "basename": "compute", + "name": "compute0", + "id": 0, + "uniq_id": 0, + "rank": -1, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0" + } + } + }, + { + "id": "1", + "metadata": { + "type": "rack", + "basename": "rack", + "name": "rack0", + "id": 0, + "uniq_id": 1, + "rank": -1, + "exclusive": true, + "unit": "", + "size": 1, + "properties": { + "rabbit": "kind-worker2", + "ssdcount": "36" + }, + "paths": { + "containment": "/compute0/rack0" + } + } + }, + { + "id": "2", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd0", + "id": 0, + "uniq_id": 2, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd0" + }, + "status": 0 + } + }, + { + "id": "3", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd1", + "id": 1, + "uniq_id": 3, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd1" + }, + "status": 0 + } + }, + { + "id": "4", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd2", + "id": 2, + "uniq_id": 4, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd2" + }, + "status": 0 + } + }, + { + "id": "5", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd3", + "id": 3, + "uniq_id": 5, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd3" + }, + "status": 0 + } + }, + { + "id": "6", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd4", + "id": 4, + "uniq_id": 6, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd4" + }, + "status": 0 + } + }, + { + "id": "7", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd5", + "id": 5, + "uniq_id": 7, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd5" + }, + "status": 0 + } + }, + { + "id": "8", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd6", + "id": 6, + "uniq_id": 8, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd6" + }, + "status": 0 + } + }, + { + "id": "9", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd7", + "id": 7, + "uniq_id": 9, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd7" + }, + "status": 0 + } + }, + { + "id": "10", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd8", + "id": 8, + "uniq_id": 10, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd8" + }, + "status": 0 + } + }, + { + "id": "11", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd9", + "id": 9, + "uniq_id": 11, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd9" + }, + "status": 0 + } + }, + { + "id": "12", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd10", + "id": 10, + "uniq_id": 12, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd10" + }, + "status": 0 + } + }, + { + "id": "13", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd11", + "id": 11, + "uniq_id": 13, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd11" + }, + "status": 0 + } + }, + { + "id": "14", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd12", + "id": 12, + "uniq_id": 14, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd12" + }, + "status": 0 + } + }, + { + "id": "15", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd13", + "id": 13, + "uniq_id": 15, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd13" + }, + "status": 0 + } + }, + { + "id": "16", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd14", + "id": 14, + "uniq_id": 16, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd14" + }, + "status": 0 + } + }, + { + "id": "17", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd15", + "id": 15, + "uniq_id": 17, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd15" + }, + "status": 0 + } + }, + { + "id": "18", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd16", + "id": 16, + "uniq_id": 18, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd16" + }, + "status": 0 + } + }, + { + "id": "19", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd17", + "id": 17, + "uniq_id": 19, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd17" + }, + "status": 0 + } + }, + { + "id": "20", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd18", + "id": 18, + "uniq_id": 20, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd18" + }, + "status": 0 + } + }, + { + "id": "21", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd19", + "id": 19, + "uniq_id": 21, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd19" + }, + "status": 0 + } + }, + { + "id": "22", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd20", + "id": 20, + "uniq_id": 22, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd20" + }, + "status": 0 + } + }, + { + "id": "23", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd21", + "id": 21, + "uniq_id": 23, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd21" + }, + "status": 0 + } + }, + { + "id": "24", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd22", + "id": 22, + "uniq_id": 24, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd22" + }, + "status": 0 + } + }, + { + "id": "25", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd23", + "id": 23, + "uniq_id": 25, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd23" + }, + "status": 0 + } + }, + { + "id": "26", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd24", + "id": 24, + "uniq_id": 26, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd24" + }, + "status": 0 + } + }, + { + "id": "27", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd25", + "id": 25, + "uniq_id": 27, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd25" + }, + "status": 0 + } + }, + { + "id": "28", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd26", + "id": 26, + "uniq_id": 28, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd26" + }, + "status": 0 + } + }, + { + "id": "29", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd27", + "id": 27, + "uniq_id": 29, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd27" + }, + "status": 0 + } + }, + { + "id": "30", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd28", + "id": 28, + "uniq_id": 30, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd28" + }, + "status": 0 + } + }, + { + "id": "31", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd29", + "id": 29, + "uniq_id": 31, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd29" + }, + "status": 0 + } + }, + { + "id": "32", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd30", + "id": 30, + "uniq_id": 32, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd30" + }, + "status": 0 + } + }, + { + "id": "33", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd31", + "id": 31, + "uniq_id": 33, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd31" + }, + "status": 0 + } + }, + { + "id": "34", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd32", + "id": 32, + "uniq_id": 34, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd32" + }, + "status": 0 + } + }, + { + "id": "35", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd33", + "id": 33, + "uniq_id": 35, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd33" + }, + "status": 0 + } + }, + { + "id": "36", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd34", + "id": 34, + "uniq_id": 36, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd34" + }, + "status": 0 + } + }, + { + "id": "37", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd35", + "id": 35, + "uniq_id": 37, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/ssd35" + }, + "status": 0 + } + }, + { + "id": "38", + "metadata": { + "type": "node", + "basename": "node", + "name": "compute-01", + "id": 1, + "uniq_id": 38, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01" + } + } + }, + { + "id": "39", + "metadata": { + "type": "core", + "basename": "core", + "name": "core0", + "id": 0, + "uniq_id": 39, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core0" + } + } + }, + { + "id": "40", + "metadata": { + "type": "core", + "basename": "core", + "name": "core1", + "id": 1, + "uniq_id": 40, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core1" + } + } + }, + { + "id": "41", + "metadata": { + "type": "core", + "basename": "core", + "name": "core2", + "id": 2, + "uniq_id": 41, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core2" + } + } + }, + { + "id": "42", + "metadata": { + "type": "core", + "basename": "core", + "name": "core3", + "id": 3, + "uniq_id": 42, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core3" + } + } + }, + { + "id": "43", + "metadata": { + "type": "core", + "basename": "core", + "name": "core4", + "id": 4, + "uniq_id": 43, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core4" + } + } + }, + { + "id": "44", + "metadata": { + "type": "core", + "basename": "core", + "name": "core5", + "id": 5, + "uniq_id": 44, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core5" + } + } + }, + { + "id": "45", + "metadata": { + "type": "core", + "basename": "core", + "name": "core6", + "id": 6, + "uniq_id": 45, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core6" + } + } + }, + { + "id": "46", + "metadata": { + "type": "core", + "basename": "core", + "name": "core7", + "id": 7, + "uniq_id": 46, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core7" + } + } + }, + { + "id": "47", + "metadata": { + "type": "core", + "basename": "core", + "name": "core8", + "id": 8, + "uniq_id": 47, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core8" + } + } + }, + { + "id": "48", + "metadata": { + "type": "core", + "basename": "core", + "name": "core9", + "id": 9, + "uniq_id": 48, + "rank": 0, + "exclusive": true, + "unit": "", + "size": 1, + "properties": {}, + "paths": { + "containment": "/compute0/rack0/compute-01/core9" + } + } + }, + { + "id": "49", + "metadata": { + "type": "rack", + "basename": "rack", + "name": "rack1", + "id": 1, + "uniq_id": 49, + "rank": -1, + "exclusive": true, + "unit": "", + "size": 1, + "properties": { + "rabbit": "kind-worker3", + "ssdcount": "36" + }, + "paths": { + "containment": "/compute0/rack1" + } + } + }, + { + "id": "50", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd0", + "id": 0, + "uniq_id": 50, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd0" + }, + "status": 0 + } + }, + { + "id": "51", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd1", + "id": 1, + "uniq_id": 51, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd1" + }, + "status": 0 + } + }, + { + "id": "52", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd2", + "id": 2, + "uniq_id": 52, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd2" + }, + "status": 0 + } + }, + { + "id": "53", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd3", + "id": 3, + "uniq_id": 53, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd3" + }, + "status": 0 + } + }, + { + "id": "54", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd4", + "id": 4, + "uniq_id": 54, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd4" + }, + "status": 0 + } + }, + { + "id": "55", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd5", + "id": 5, + "uniq_id": 55, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd5" + }, + "status": 0 + } + }, + { + "id": "56", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd6", + "id": 6, + "uniq_id": 56, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd6" + }, + "status": 0 + } + }, + { + "id": "57", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd7", + "id": 7, + "uniq_id": 57, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd7" + }, + "status": 0 + } + }, + { + "id": "58", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd8", + "id": 8, + "uniq_id": 58, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd8" + }, + "status": 0 + } + }, + { + "id": "59", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd9", + "id": 9, + "uniq_id": 59, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd9" + }, + "status": 0 + } + }, + { + "id": "60", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd10", + "id": 10, + "uniq_id": 60, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd10" + }, + "status": 0 + } + }, + { + "id": "61", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd11", + "id": 11, + "uniq_id": 61, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd11" + }, + "status": 0 + } + }, + { + "id": "62", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd12", + "id": 12, + "uniq_id": 62, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd12" + }, + "status": 0 + } + }, + { + "id": "63", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd13", + "id": 13, + "uniq_id": 63, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd13" + }, + "status": 0 + } + }, + { + "id": "64", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd14", + "id": 14, + "uniq_id": 64, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd14" + }, + "status": 0 + } + }, + { + "id": "65", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd15", + "id": 15, + "uniq_id": 65, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd15" + }, + "status": 0 + } + }, + { + "id": "66", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd16", + "id": 16, + "uniq_id": 66, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd16" + }, + "status": 0 + } + }, + { + "id": "67", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd17", + "id": 17, + "uniq_id": 67, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd17" + }, + "status": 0 + } + }, + { + "id": "68", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd18", + "id": 18, + "uniq_id": 68, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd18" + }, + "status": 0 + } + }, + { + "id": "69", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd19", + "id": 19, + "uniq_id": 69, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd19" + }, + "status": 0 + } + }, + { + "id": "70", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd20", + "id": 20, + "uniq_id": 70, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd20" + }, + "status": 0 + } + }, + { + "id": "71", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd21", + "id": 21, + "uniq_id": 71, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd21" + }, + "status": 0 + } + }, + { + "id": "72", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd22", + "id": 22, + "uniq_id": 72, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd22" + }, + "status": 0 + } + }, + { + "id": "73", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd23", + "id": 23, + "uniq_id": 73, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd23" + }, + "status": 0 + } + }, + { + "id": "74", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd24", + "id": 24, + "uniq_id": 74, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd24" + }, + "status": 0 + } + }, + { + "id": "75", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd25", + "id": 25, + "uniq_id": 75, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd25" + }, + "status": 0 + } + }, + { + "id": "76", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd26", + "id": 26, + "uniq_id": 76, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd26" + }, + "status": 0 + } + }, + { + "id": "77", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd27", + "id": 27, + "uniq_id": 77, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd27" + }, + "status": 0 + } + }, + { + "id": "78", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd28", + "id": 28, + "uniq_id": 78, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd28" + }, + "status": 0 + } + }, + { + "id": "79", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd29", + "id": 29, + "uniq_id": 79, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd29" + }, + "status": 0 + } + }, + { + "id": "80", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd30", + "id": 30, + "uniq_id": 80, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd30" + }, + "status": 0 + } + }, + { + "id": "81", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd31", + "id": 31, + "uniq_id": 81, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd31" + }, + "status": 0 + } + }, + { + "id": "82", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd32", + "id": 32, + "uniq_id": 82, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd32" + }, + "status": 0 + } + }, + { + "id": "83", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd33", + "id": 33, + "uniq_id": 83, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd33" + }, + "status": 0 + } + }, + { + "id": "84", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd34", + "id": 34, + "uniq_id": 84, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd34" + }, + "status": 0 + } + }, + { + "id": "85", + "metadata": { + "type": "ssd", + "basename": "ssd", + "name": "ssd35", + "id": 35, + "uniq_id": 85, + "rank": -1, + "exclusive": true, + "unit": "GiB", + "size": 1024, + "properties": {}, + "paths": { + "containment": "/compute0/rack1/ssd35" + }, + "status": 0 + } + } + ], + "edges": [ + { + "source": "0", + "target": "1", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "2", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "3", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "4", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "5", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "6", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "7", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "8", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "9", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "10", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "11", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "12", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "13", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "14", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "15", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "16", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "17", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "18", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "19", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "20", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "21", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "22", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "23", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "24", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "25", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "26", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "27", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "28", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "29", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "30", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "31", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "32", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "33", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "34", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "35", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "36", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "37", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "1", + "target": "38", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "39", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "40", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "41", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "42", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "43", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "44", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "45", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "46", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "47", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "38", + "target": "48", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "0", + "target": "49", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "50", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "51", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "52", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "53", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "54", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "55", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "56", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "57", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "58", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "59", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "60", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "61", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "62", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "63", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "64", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "65", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "66", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "67", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "68", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "69", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "70", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "71", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "72", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "73", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "74", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "75", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "76", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "77", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "78", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "79", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "80", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "81", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "82", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "83", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "84", + "directed": true, + "metadata": { + "subsystem": "containment" + } + }, + { + "source": "49", + "target": "85", + "directed": true, + "metadata": { + "subsystem": "containment" + } + } + ] + } + } +} diff --git a/t/data/resource/jobspecs/issues/issue1284-noslot.json b/t/data/resource/jobspecs/issues/issue1284-noslot.json new file mode 100644 index 000000000..846e031dd --- /dev/null +++ b/t/data/resource/jobspecs/issues/issue1284-noslot.json @@ -0,0 +1,46 @@ +{ + "resources": [ + { + "type": "node", + "count": 1, + "exclusive": true, + "with": [ + { + "type": "slot", + "count": 1, + "with": [ + { + "type": "core", + "count": 1 + } + ], + "label": "task" + } + ] + }, + { + "type": "ssd", + "count": 20480, + "exclusive": true + } + ], + "tasks": [ + { + "command": [ + "hostname" + ], + "slot": "task", + "count": { + "per_slot": 1 + } + } + ], + "attributes": { + "system": { + "duration": 0, + "environment": {}, + "shell": {} + } + }, + "version": 1 +} diff --git a/t/data/resource/jobspecs/issues/issue1284.json b/t/data/resource/jobspecs/issues/issue1284.json new file mode 100644 index 000000000..59ace6f26 --- /dev/null +++ b/t/data/resource/jobspecs/issues/issue1284.json @@ -0,0 +1,53 @@ +{ + "resources": [ + { + "type": "slot", + "count": 1, + "label": "default", + "with": [ + { + "type": "node", + "count": 1, + "exclusive": true, + "with": [ + { + "type": "slot", + "count": 1, + "with": [ + { + "type": "core", + "count": 1 + } + ], + "label": "task" + } + ] + }, + { + "type": "ssd", + "count": 20480, + "exclusive": true + } + ] + } + ], + "tasks": [ + { + "command": [ + "hostname" + ], + "slot": "task", + "count": { + "per_slot": 1 + } + } + ], + "attributes": { + "system": { + "duration": 0, + "environment": {}, + "shell": {} + } + }, + "version": 1 +} diff --git a/t/t1027-rv1-partial-release-brokerless-resources.t b/t/t1027-rv1-partial-release-brokerless-resources.t new file mode 100755 index 000000000..b8d6313ee --- /dev/null +++ b/t/t1027-rv1-partial-release-brokerless-resources.t @@ -0,0 +1,193 @@ +#!/bin/sh +# +test_description='Ensure fluxion cancels resources that are not associated with a broker rank' + +# See, e.g. issue #1284 + +. `dirname $0`/sharness.sh + +if test_have_prereq ASAN; then + skip_all='skipping issues tests under AddressSanitizer' + test_done +fi +SIZE=1 +test_under_flux ${SIZE} + + +test_expect_success 'an ssd jobspec can be allocated' ' + flux module remove sched-simple && + flux module remove resource && + flux config load <