Restructure code in preparation for pthread group support. (#238)

Signed-off-by: Samuel K. Gutierrez <samuel@lanl.gov>

src/CMakeLists.txt

@@ -31,6 +31,7 @@ add_library(
       qvi-task.h
       qvi-pthread.h
       qvi-group.h
+      qvi-subgroup.h
       qvi-group-pthread.h
       qvi-map.h
       qvi-scope.h

src/qvi-group.h

@@ -34,12 +34,12 @@ struct qvi_group_s : qvi_refc_s {
     /** Returns pointer to the caller's task information. */
     virtual qvi_task_t *
     task(void) = 0;
-    /** Returns the caller's group rank. */
-    virtual int
-    rank(void) = 0;
     /** Returns the number of members in this group. */
     virtual int
     size(void) = 0;
+    /** Returns the caller's group rank. */
+    virtual int
+    rank(void) = 0;
     /** Performs node-local group barrier. */
     virtual int
     barrier(void) = 0;

src/qvi-omp.cc

@@ -18,54 +18,11 @@
  */
 
 #include "qvi-omp.h"
+#include "qvi-subgroup.h"
 #include "qvi-bbuff.h"
 #include "qvi-utils.h"
 #include <omp.h>
 
-/**
- * Internal data structure for rank and size computing.
- */
-struct qvi_omp_color_key_rank_s {
-    int color = 0;
-    int key = 0;
-    int rank = 0;
-
-    static bool
-    by_color(
-        const qvi_omp_color_key_rank_s &a,
-        const qvi_omp_color_key_rank_s &b
-    ) {
-        return a.color < b.color;
-    }
-
-    static bool
-    by_key(
-        const qvi_omp_color_key_rank_s &a,
-        const qvi_omp_color_key_rank_s &b
-    ) {
-        // If colors are the same, sort by key.
-        return a.color == b.color && a.key < b.key;
-    }
-
-    static bool
-    by_rank(
-        const qvi_omp_color_key_rank_s &a,
-        const qvi_omp_color_key_rank_s &b
-    ) {
-        // If colors and keys are the same, sort by rank.
-        return a.color == b.color && a.key == b.key && a.rank < b.rank;
-    }
-};
-
-struct qvi_omp_subgroup_info_s {
-    /** Number of sub-groups created from split. */
-    int ngroups = 0;
-    /** Number of members in this sub-group. */
-    int size = 0;
-    /** My rank in this sub-group. */
-    int rank = 0;
-};
-
 struct qvi_omp_group_s {
     /** Group size. */
     int size = 0;
@@ -124,14 +81,14 @@ qvi_get_subgroup_info(
     qvi_omp_group_t *parent,
     int color,
     int key,
-    qvi_omp_subgroup_info_s *sginfo
+    qvi_subgroup_info_s *sginfo
 ) {
     const int size = parent->size;
     const int rank = parent->rank;
-    qvi_omp_color_key_rank_s *ckrs = nullptr;
+    qvi_subgroup_color_key_rank_s *ckrs = nullptr;
 
     #pragma omp single copyprivate(ckrs)
-    ckrs = new qvi_omp_color_key_rank_s[size];
+    ckrs = new qvi_subgroup_color_key_rank_s[size];
     // Gather colors and keys from ALL threads.
     ckrs[rank].color = color;
     ckrs[rank].key = key;
@@ -146,9 +103,9 @@ qvi_get_subgroup_info(
         // Sort the color/key/rank array. First according to color, then by key,
         // but in the same color realm. If color and key are identical, sort by
         // the rank from given group.
-        std::sort(ckrs, ckrs + size, qvi_omp_color_key_rank_s::by_color);
-        std::sort(ckrs, ckrs + size, qvi_omp_color_key_rank_s::by_key);
-        std::sort(ckrs, ckrs + size, qvi_omp_color_key_rank_s::by_rank);
+        std::sort(ckrs, ckrs + size, qvi_subgroup_color_key_rank_s::by_color);
+        std::sort(ckrs, ckrs + size, qvi_subgroup_color_key_rank_s::by_key);
+        std::sort(ckrs, ckrs + size, qvi_subgroup_color_key_rank_s::by_rank);
         // Calculate the number of distinct colors provided.
         std::set<int> color_set;
         for (int i = 0; i < size; ++i) {
@@ -192,14 +149,13 @@ qvi_omp_group_create_from_split(
 ) {
     qvi_omp_group_t *ichild = nullptr;
 
-    qvi_omp_subgroup_info_s sginfo;
+    qvi_subgroup_info_s sginfo;
     int rc = qvi_get_subgroup_info(
         parent, color, key, &sginfo
     );
-    if (rc != QV_SUCCESS) goto out;
-
-    rc = qvi_new(&ichild, sginfo.size, sginfo.rank);
-out:
+    if (qvi_likely(rc == QV_SUCCESS)) {
+        rc = qvi_new(&ichild, sginfo.size, sginfo.rank);
+    }
     if (rc != QV_SUCCESS) {
         qvi_delete(&ichild);
     }

src/qvi-omp.h

@@ -14,7 +14,7 @@
  */
 
 /**
- * @file qvi-thread.h
+ * @file qvi-omp.h
  */
 
 #ifndef QVI_OMP_H

src/qvi-pthread.cc

@@ -43,8 +43,8 @@ qvi_pthread_group_s::call_first_from_pthread_create(
         std::sort(group->m_tids.begin(), group->m_tids.end());
 
         for (int i = 0; i < group->m_size; ++i) {
-            const pid_t tid = group->m_tids[i];
-            group->m_tid2rank.insert({tid, i});
+            const pid_t tidi = group->m_tids[i];
+            group->m_tid2rank.insert({tidi, i});
         }
         pthread_barrier_wait(&group->m_barrier);
     }

src/qvi-subgroup.h

@@ -0,0 +1,78 @@
+/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2022-2024 Triad National Security, LLC
+ *                         All rights reserved.
+ *
+ * Copyright (c) 2022-2024 Inria
+ *                         All rights reserved.
+ *
+ * Copyright (c) 2022-2024 Bordeaux INP
+ *                         All rights reserved.
+ *
+ * This file is part of the quo-vadis project. See the LICENSE file at the
+ * top-level directory of this distribution.
+ */
+
+/**
+ * @file qvi-subgroup.h
+ */
+
+#ifndef QVI_SUBGROUP_H
+#define QVI_SUBGROUP_H
+
+#include "qvi-common.h" // IWYU pragma: keep
+
+/**
+ * Stores sub-group information for infrastructure that
+ * doesn't have native support for creating sub-groups.
+ */
+struct qvi_subgroup_info_s {
+    /** Number of sub-groups created from split. */
+    int ngroups = 0;
+    /** Number of members in this sub-group. */
+    int size = 0;
+    /** My rank in this sub-group. */
+    int rank = 0;
+};
+
+/**
+ * Provides supporting infrastructure for creating
+ * sub-groups based on color, key, and rank.
+ */
+struct qvi_subgroup_color_key_rank_s {
+    int color = 0;
+    int key = 0;
+    int rank = 0;
+
+    static bool
+    by_color(
+        const qvi_subgroup_color_key_rank_s &a,
+        const qvi_subgroup_color_key_rank_s &b
+    ) {
+        return a.color < b.color;
+    }
+
+    static bool
+    by_key(
+        const qvi_subgroup_color_key_rank_s &a,
+        const qvi_subgroup_color_key_rank_s &b
+    ) {
+        // If colors are the same, sort by key.
+        return a.color == b.color && a.key < b.key;
+    }
+
+    static bool
+    by_rank(
+        const qvi_subgroup_color_key_rank_s &a,
+        const qvi_subgroup_color_key_rank_s &b
+    ) {
+        // If colors and keys are the same, sort by rank.
+        return a.color == b.color && a.key == b.key && a.rank < b.rank;
+    }
+};
+
+#endif
+
+/*
+ * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
+ */