From a90bb9647b24bc4a5cdae47e2d304285a947cd3c Mon Sep 17 00:00:00 2001
From: Danny Auble <da@schedmd.com>
Date: Tue, 16 Aug 2011 14:49:54 -0700
Subject: [PATCH] BLUGENE - Added logic to keep track of the ba_geo system
inside the block_allocator since it is different than the
node_record_table_ptr.
---
.../select/bluegene/ba_bgq/block_allocator.c | 104 ++++++++++--------
src/plugins/select/bluegene/ba_common.c | 73 ++++++------
src/plugins/select/bluegene/ba_common.h | 20 +++-
3 files changed, 111 insertions(+), 86 deletions(-)
diff --git a/src/plugins/select/bluegene/ba_bgq/block_allocator.c b/src/plugins/select/bluegene/ba_bgq/block_allocator.c
index b2936d38d71..703d71b00b0 100644
--- a/src/plugins/select/bluegene/ba_bgq/block_allocator.c
+++ b/src/plugins/select/bluegene/ba_bgq/block_allocator.c
@@ -69,7 +69,7 @@ static ba_geo_system_t *ba_main_geo_system = NULL;
static ba_geo_system_t *ba_mp_geo_system = NULL;
static uint16_t *deny_pass = NULL;
static ba_nc_coords_t g_nc_coords[16];
-
+static ba_mp_t **ba_main_grid_array = NULL;
/* increment Y -> Z -> A -> X -> E
* used for doing nodecard coords */
static int ba_nc_dim_order[5] = {Y, Z, A, X, E};
@@ -124,44 +124,6 @@ extern void ba_create_system()
if (ba_main_grid)
ba_destroy_system();
- ba_main_grid = (ba_mp_t****)
- xmalloc(sizeof(ba_mp_t***) * DIM_SIZE[A]);
- for (a = 0; a < DIM_SIZE[A]; a++) {
- ba_main_grid[a] = (ba_mp_t***)
- xmalloc(sizeof(ba_mp_t**) * DIM_SIZE[X]);
- for (x = 0; x < DIM_SIZE[X]; x++) {
- ba_main_grid[a][x] = (ba_mp_t**)
- xmalloc(sizeof(ba_mp_t*) * DIM_SIZE[Y]);
- for (y = 0; y < DIM_SIZE[Y]; y++) {
- ba_main_grid[a][x][y] = (ba_mp_t*)
- xmalloc(sizeof(ba_mp_t) * DIM_SIZE[Z]);
- for (z = 0; z < DIM_SIZE[Z]; z++) {
- ba_mp_t *ba_mp = &ba_main_grid
- [a][x][y][z];
- ba_mp->coord[A] = a;
- ba_mp->coord[X] = x;
- ba_mp->coord[Y] = y;
- ba_mp->coord[Z] = z;
-
- snprintf(ba_mp->coord_str,
- sizeof(ba_mp->coord_str),
- "%c%c%c%c",
- alpha_num[ba_mp->coord[A]],
- alpha_num[ba_mp->coord[X]],
- alpha_num[ba_mp->coord[Y]],
- alpha_num[ba_mp->coord[Z]]);
- ba_setup_mp(ba_mp, true, false);
- ba_mp->state = NODE_STATE_IDLE;
- /* This might get changed
- later, but just incase set
- it up here.
- */
- ba_mp->index = i++;
- }
- }
- }
- }
-
/* build all the possible geos for the mid planes */
ba_main_geo_system = xmalloc(sizeof(ba_geo_system_t));
ba_main_geo_system->dim_count = SYSTEM_DIMENSIONS;
@@ -188,8 +150,6 @@ extern void ba_create_system()
ba_create_geo_table(ba_mp_geo_system);
//ba_print_geo_table(ba_mp_geo_system);
- _setup_next_mps(A, coords);
-
/* Now set it up to mark the corners of each nodecard. This
is used if running a sub-block job on a small block later.
*/
@@ -244,6 +204,58 @@ extern void ba_create_system()
/* alpha_num[g_nc_coords[i].end[E]]); */
_increment_nc_coords(0, mp_coords, ba_mp_geo_system->dim_size);
}
+
+ /* Set up a flat array to be used in conjunction with the
+ ba_geo system.
+ */
+ ba_main_grid_array = xmalloc(sizeof(ba_mp_t *) *
+ ba_main_geo_system->total_size);
+
+ ba_main_grid = (ba_mp_t****)
+ xmalloc(sizeof(ba_mp_t***) * DIM_SIZE[A]);
+ for (a = 0; a < DIM_SIZE[A]; a++) {
+ ba_main_grid[a] = (ba_mp_t***)
+ xmalloc(sizeof(ba_mp_t**) * DIM_SIZE[X]);
+ for (x = 0; x < DIM_SIZE[X]; x++) {
+ ba_main_grid[a][x] = (ba_mp_t**)
+ xmalloc(sizeof(ba_mp_t*) * DIM_SIZE[Y]);
+ for (y = 0; y < DIM_SIZE[Y]; y++) {
+ ba_main_grid[a][x][y] = (ba_mp_t*)
+ xmalloc(sizeof(ba_mp_t) * DIM_SIZE[Z]);
+ for (z = 0; z < DIM_SIZE[Z]; z++) {
+ ba_mp_t *ba_mp = &ba_main_grid
+ [a][x][y][z];
+ ba_mp->coord[A] = a;
+ ba_mp->coord[X] = x;
+ ba_mp->coord[Y] = y;
+ ba_mp->coord[Z] = z;
+
+ snprintf(ba_mp->coord_str,
+ sizeof(ba_mp->coord_str),
+ "%c%c%c%c",
+ alpha_num[ba_mp->coord[A]],
+ alpha_num[ba_mp->coord[X]],
+ alpha_num[ba_mp->coord[Y]],
+ alpha_num[ba_mp->coord[Z]]);
+ ba_setup_mp(ba_mp, true, false);
+ ba_mp->state = NODE_STATE_IDLE;
+ /* This might get changed
+ later, but just incase set
+ it up here.
+ */
+ ba_mp->index = i++;
+ ba_mp->ba_geo_index =
+ ba_node_xlate_to_1d(
+ ba_mp->coord,
+ ba_main_geo_system);
+ ba_main_grid_array[ba_mp->ba_geo_index]
+ = ba_mp;
+ }
+ }
+ }
+ }
+
+ _setup_next_mps(A, coords);
}
/** */
@@ -251,6 +263,8 @@ extern void ba_destroy_system(void)
{
int a, x, y;
+ xfree(ba_main_grid_array);
+
if (ba_main_grid) {
for (a=0; a<DIM_SIZE[A]; a++) {
for (x = 0; x < DIM_SIZE[X]; x++) {
@@ -556,7 +570,8 @@ extern int remove_block(List mps, bool is_small)
if (curr_ba_mp->used) {
ba_mp->used &= (~BA_MP_USED_TRUE);
if (ba_mp->used == BA_MP_USED_FALSE)
- bit_clear(ba_main_mp_bitmap, ba_mp->index);
+ bit_clear(ba_main_mp_bitmap,
+ ba_mp->ba_geo_index);
}
ba_mp->used &= (~BA_MP_USED_ALTERED_PASS);
@@ -675,8 +690,9 @@ extern int check_and_set_mp_list(List mps)
if (ba_mp->used) {
curr_ba_mp->used = ba_mp->used;
- xassert(!bit_test(ba_main_mp_bitmap, ba_mp->index));
- bit_set(ba_main_mp_bitmap, ba_mp->index);
+ xassert(!bit_test(ba_main_mp_bitmap,
+ ba_mp->ba_geo_index));
+ bit_set(ba_main_mp_bitmap, ba_mp->ba_geo_index);
}
if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
diff --git a/src/plugins/select/bluegene/ba_common.c b/src/plugins/select/bluegene/ba_common.c
index 944a0c28e70..e83c59f67c8 100644
--- a/src/plugins/select/bluegene/ba_common.c
+++ b/src/plugins/select/bluegene/ba_common.c
@@ -145,24 +145,6 @@ static bool _incr_geo(int *geo, ba_geo_system_t *my_geo_system)
return false;
}
-/* Translate a multi-dimension coordinate (3-D, 4-D, 5-D, etc.) into a 1-D
- * offset in the cnode* bitmap */
-static void _ba_node_xlate_to_1d(int *offset_1d, int *full_offset,
- ba_geo_system_t *my_geo_system)
-{
- int i, map_offset;
-
- xassert(offset_1d);
- xassert(full_offset);
- i = my_geo_system->dim_count - 1;
- map_offset = full_offset[i];
- for (i-- ; i >= 0; i--) {
- map_offset *= my_geo_system->dim_size[i];
- map_offset += full_offset[i];
- }
- *offset_1d = map_offset;
-}
-
#if DISPLAY_FULL_DIM
/* Translate a 1-D offset in the cnode bitmap to a multi-dimension
* coordinate (3-D, 4-D, 5-D, etc.) */
@@ -180,13 +162,11 @@ static void _ba_node_xlate_from_1d(int offset_1d, int *full_offset,
}
#endif
-static int _ba_node_map_set_range_internal(int level, int *coords,
+static int _ba_node_map_set_range_internal(int level, uint16_t *coords,
int *start_offset, int *end_offset,
bitstr_t *node_bitmap,
ba_geo_system_t *my_geo_system)
{
- int offset_1d;
-
xassert(my_geo_system);
if (level > my_geo_system->dim_count)
@@ -206,8 +186,7 @@ static int _ba_node_map_set_range_internal(int level, int *coords,
return 1;
}
- _ba_node_xlate_to_1d(&offset_1d, coords, my_geo_system);
- bit_set(node_bitmap, offset_1d);
+ ba_node_map_set(node_bitmap, coords, my_geo_system);
return 1;
}
@@ -345,7 +324,7 @@ static bitstr_t * _test_geo(bitstr_t *node_bitmap,
{
int i;
bitstr_t *alloc_node_bitmap;
- int offset[my_geo_system->dim_count];
+ uint16_t offset[my_geo_system->dim_count];
alloc_node_bitmap = bit_alloc(my_geo_system->total_size);
memset(offset, 0, sizeof(offset));
@@ -359,7 +338,8 @@ static bitstr_t * _test_geo(bitstr_t *node_bitmap,
}
/* Test if this coordinate is available for use */
if (i >= my_geo_system->dim_count) {
- if (ba_node_map_test(node_bitmap,offset,my_geo_system))
+ if (ba_node_map_test(
+ node_bitmap, offset, my_geo_system))
break; /* not available */
/* Set it in our bitmap for this job */
ba_node_map_set(alloc_node_bitmap, offset,
@@ -496,11 +476,12 @@ static void _internal_removable_set_mps(int level, bitstr_t *bitmap,
if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
info("can't use %s", curr_mp->coord_str);
curr_mp->used |= BA_MP_USED_TEMP;
- bit_set(ba_main_mp_bitmap, curr_mp->index);
+ bit_set(ba_main_mp_bitmap, curr_mp->ba_geo_index);
} else {
curr_mp->used &= (~BA_MP_USED_TEMP);
if (curr_mp->used == BA_MP_USED_FALSE)
- bit_clear(ba_main_mp_bitmap, curr_mp->index);
+ bit_clear(ba_main_mp_bitmap,
+ curr_mp->ba_geo_index);
}
}
}
@@ -527,7 +508,7 @@ static void _internal_reset_ba_system(int level, uint16_t *coords,
if (!curr_mp)
return;
ba_setup_mp(curr_mp, track_down_mps, false);
- bit_clear(ba_main_mp_bitmap, curr_mp->index);
+ bit_clear(ba_main_mp_bitmap, curr_mp->ba_geo_index);
}
#if defined HAVE_BG_FILES
@@ -868,6 +849,7 @@ extern int unpack_ba_mp(ba_mp_t **ba_mp_pptr,
if (!orig_mp)
goto unpack_error;
ba_mp->index = orig_mp->index;
+ ba_mp->ba_geo_index = orig_mp->ba_geo_index;
return SLURM_SUCCESS;
@@ -1174,13 +1156,10 @@ extern void ba_node_map_free(bitstr_t *node_bitmap,
* IN full_offset - N-dimension zero-origin offset to set
* IN my_geo_system - system geometry specification
*/
-extern void ba_node_map_set(bitstr_t *node_bitmap, int *full_offset,
+extern void ba_node_map_set(bitstr_t *node_bitmap, uint16_t *full_offset,
ba_geo_system_t *my_geo_system)
{
- int offset_1d;
-
- _ba_node_xlate_to_1d(&offset_1d, full_offset, my_geo_system);
- bit_set(node_bitmap, offset_1d);
+ bit_set(node_bitmap, ba_node_xlate_to_1d(full_offset, my_geo_system));
}
/*
@@ -1194,7 +1173,7 @@ extern void ba_node_map_set_range(bitstr_t *node_bitmap,
int *start_offset, int *end_offset,
ba_geo_system_t *my_geo_system)
{
- int coords[5];
+ uint16_t coords[HIGHEST_DIMENSIONS];
_ba_node_map_set_range_internal(0, coords, start_offset, end_offset,
node_bitmap, my_geo_system);
@@ -1206,13 +1185,11 @@ extern void ba_node_map_set_range(bitstr_t *node_bitmap,
* IN full_offset - N-dimension zero-origin offset to test
* IN my_geo_system - system geometry specification
*/
-extern int ba_node_map_test(bitstr_t *node_bitmap, int *full_offset,
+extern int ba_node_map_test(bitstr_t *node_bitmap, uint16_t *full_offset,
ba_geo_system_t *my_geo_system)
{
- int offset_1d;
-
- _ba_node_xlate_to_1d(&offset_1d, full_offset, my_geo_system);
- return bit_test(node_bitmap, offset_1d);
+ return bit_test(node_bitmap,
+ ba_node_xlate_to_1d(full_offset, my_geo_system));
}
/*
@@ -1398,6 +1375,24 @@ extern int ba_geo_test_all(bitstr_t *node_bitmap,
return rc;
}
+/* Translate a multi-dimension coordinate (3-D, 4-D, 5-D, etc.) into a 1-D
+ * offset in the cnode* bitmap */
+extern int ba_node_xlate_to_1d(uint16_t *full_offset,
+ ba_geo_system_t *my_geo_system)
+{
+ int i, map_offset;
+
+ xassert(full_offset);
+ xassert(my_geo_system);
+ i = my_geo_system->dim_count - 1;
+ map_offset = full_offset[i];
+ for (i-- ; i >= 0; i--) {
+ map_offset *= my_geo_system->dim_size[i];
+ map_offset += full_offset[i];
+ }
+ return map_offset;
+}
+
/*
* Used to set all midplanes in a special used state except the ones
* we are able to use in a new allocation.
diff --git a/src/plugins/select/bluegene/ba_common.h b/src/plugins/select/bluegene/ba_common.h
index 8b9344f9ba5..7cfd06a7eab 100644
--- a/src/plugins/select/bluegene/ba_common.h
+++ b/src/plugins/select/bluegene/ba_common.h
@@ -148,13 +148,17 @@ typedef struct block_allocator_mp {
ba_switch_t alter_switch[HIGHEST_DIMENSIONS];
/* a switch for each dimensions */
ba_switch_t axis_switch[HIGHEST_DIMENSIONS];
+ /* index into the ba_main_grid_array (BGQ) used for easy look
+ * up of the miplane in that system */
+ uint32_t ba_geo_index;
/* Bitmap of available cnodes */
bitstr_t *cnode_bitmap;
/* coordinates of midplane */
uint16_t coord[HIGHEST_DIMENSIONS];
/* coordinates of midplane in str format */
char coord_str[HIGHEST_DIMENSIONS+1];
- /* midplane index used for easy look up of the miplane */
+ /* index into the node_record_table_ptr used for easy look up
+ * of the miplane in that system */
uint32_t index;
/* rack-midplane location. */
char *loc;
@@ -284,7 +288,7 @@ extern void ba_node_map_free(bitstr_t *node_bitmap,
* IN full_offset - N-dimension zero-origin offset to set
* IN my_geo_system - system geometry specification
*/
-extern void ba_node_map_set(bitstr_t *node_bitmap, int *full_offset,
+extern void ba_node_map_set(bitstr_t *node_bitmap, uint16_t *full_offset,
ba_geo_system_t *my_geo_system);
/*
@@ -304,7 +308,7 @@ extern void ba_node_map_set_range(bitstr_t *node_bitmap,
* IN full_offset - N-dimension zero-origin offset to test
* IN my_geo_system - system geometry specification
*/
-extern int ba_node_map_test(bitstr_t *node_bitmap, int *full_offset,
+extern int ba_node_map_test(bitstr_t *node_bitmap, uint16_t *full_offset,
ba_geo_system_t *my_geo_system);
/*
@@ -386,6 +390,16 @@ extern int ba_geo_test_all(bitstr_t *node_bitmap,
uint16_t *start_pos, int *scan_offset,
bool deny_wrap);
+/* Translate a multi-dimension coordinate (3-D, 4-D, 5-D, etc.) into a 1-D
+ * offset in the ba_geo_system_t bitmap
+ *
+ * IN full_offset - N-dimension zero-origin offset to test
+ * IN my_geo_system - system geometry specification
+ * RET - 1-D offset
+ */
+extern int ba_node_xlate_to_1d(uint16_t *full_offset,
+ ba_geo_system_t *my_geo_system);
+
/*
* Used to set all midplanes in a special used state except the ones
* we are able to use in a new allocation.
--
GitLab