Checkpoint first cut at pthread group support.

Still needs plenty of work.

Signed-off-by: Samuel K. Gutierrez <samuel@lanl.gov>

src/CMakeLists.txt

@@ -29,7 +29,9 @@ add_library(
       qvi-rmi.h
       qvi-bbuff-rmi.h
       qvi-task.h
+      qvi-pthread.h
       qvi-group.h
+      qvi-group-pthread.h
       qvi-map.h
       qvi-scope.h
       qvi-log.cc
@@ -41,6 +43,9 @@ add_library(
       qvi-hwpool.cc
       qvi-rmi.cc
       qvi-task.cc
+      qvi-group.cc
+      qvi-pthread.cc
+      qvi-group-pthread.cc
       qvi-map.cc
       qvi-scope.cc
       quo-vadis.cc

src/quo-vadis-pthread.cc

@@ -19,11 +19,11 @@
 
 #include "qvi-common.h" // IWYU pragma: keep
 #include "quo-vadis-pthread.h"
+#include "qvi-pthread.h"
+#include "qvi-group-pthread.h"
 #include "qvi-scope.h"
 #include "qvi-utils.h"
 
-typedef void *(*qvi_pthread_routine_fun_ptr_t)(void *);
-
 struct qvi_pthread_args_s {
     qv_scope_t *scope = nullptr;
     qvi_pthread_routine_fun_ptr_t th_routine = nullptr;
@@ -66,7 +66,7 @@ qv_pthread_scope_split(
         return QV_ERR_INVLD_ARG;
     }
     try {
-        return qvi_scope_ksplit(
+        return qvi_scope_thsplit(
             scope, npieces, color_array, nthreads,
             QV_HW_OBJ_LAST, subscope
         );
@@ -86,7 +86,7 @@ qv_pthread_scope_split_at(
         return QV_ERR_INVLD_ARG;
     }
     try {
-        return qvi_scope_ksplit_at(
+        return qvi_scope_thsplit_at(
             scope, type, color_array, nthreads, subscopes
         );
     }
@@ -103,12 +103,26 @@ qv_pthread_create(
 ) {
     // Memory will be freed in qv_pthread_routine to avoid memory leaks.
     qvi_pthread_args_s *arg_ptr = nullptr;
-    const int rc = qvi_new(&arg_ptr, scope, thread_routine, arg);
+    int rc = qvi_new(&arg_ptr, scope, thread_routine, arg);
     // Since this is meant to behave similarly to
     // pthread_create(), return a reasonable errno.
     if (qvi_unlikely(rc != QV_SUCCESS)) return ENOMEM;
+    // TODO(skg) Cleanup.
+    qvi_pthread_group_pthread_create_args_s *cargs = nullptr;
+    rc = qvi_new(&cargs);
+    if (qvi_unlikely(rc != QV_SUCCESS)) {
+        qvi_delete(&arg_ptr);
+        return ENOMEM;
+    }
+    // TODO(skg) Cleanup.
+    auto gt = dynamic_cast<qvi_group_pthread_t *>(qvi_scope_group_get(scope));
+    cargs->group = gt->thgroup;
+    cargs->th_routine = qvi_pthread_routine;
+    cargs->th_routine_argp = arg_ptr;
 
-    return pthread_create(thread, attr, qvi_pthread_routine, arg_ptr);
+    return pthread_create(
+        thread, attr, qvi_pthread_group_s::call_first_from_pthread_create, cargs
+    );
 }
 
 int
@@ -120,7 +134,7 @@ qv_pthread_scopes_free(
         return QV_ERR_INVLD_ARG;
     }
     try {
-        qvi_scope_kfree(&scopes, nscopes);
+        qvi_scope_thfree(&scopes, nscopes);
         return QV_SUCCESS;
     }
     qvi_catch_and_return();

src/qvi-common.h

@@ -47,6 +47,7 @@
 #include "qvi-log.h"
 #include <chrono>
 #include <map>
+#include <mutex>
 #include <new>
 #include <numeric>
 #include <set>

src/qvi-group-omp.h

@@ -63,6 +63,15 @@ struct qvi_group_omp_s : public qvi_group_s {
         qvi_group_s **child
     );
 
+    virtual int
+    thsplit(
+        int,
+        qvi_group_s **
+    ) {
+        // TODO(skg) Need to test this.
+        return QV_ERR_NOT_SUPPORTED;
+    }
+
     virtual int
     split(
         int color,

src/qvi-group-pthread.cc

@@ -0,0 +1,54 @@
+/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2020-2024 Triad National Security, LLC
+ *                         All rights reserved.
+ *
+ * This file is part of the quo-vadis project. See the LICENSE file at the
+ * top-level directory of this distribution.
+ */
+
+/**
+ * @file qvi-group-pthread.cc
+ */
+
+#include "qvi-group-pthread.h"
+#include "qvi-utils.h"
+
+int
+qvi_group_pthread_s::self(
+    qvi_group_t **child
+) {
+    constexpr int group_size = 1;
+    qvi_group_pthread_t *ichild = nullptr;
+    int rc = qvi_new(&ichild);
+    if (rc != QV_SUCCESS) goto out;
+    // Create a group containing a single thread.
+    rc = qvi_new(&ichild->thgroup, group_size);
+out:
+    if (rc != QV_SUCCESS) {
+        qvi_delete(&ichild);
+    }
+    *child = ichild;
+    return rc;
+}
+
+int
+qvi_group_pthread_s::split(
+    int,
+    int,
+    qvi_group_t **
+) {
+    // TODO(skg)
+    return QV_ERR_NOT_SUPPORTED;
+}
+
+qvi_zgroup_pthread_s::qvi_zgroup_pthread_s(
+    int group_size
+) {
+    int rc = qvi_new(&thgroup, group_size);
+    if (rc != QV_SUCCESS) throw qvi_runtime_error();
+}
+
+/*
+ * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
+ */

src/qvi-group-pthread.h

@@ -0,0 +1,123 @@
+/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2020-2024 Triad National Security, LLC
+ *                         All rights reserved.
+ *
+ * This file is part of the quo-vadis project. See the LICENSE file at the
+ * top-level directory of this distribution.
+ */
+
+/**
+ * @file qvi-group-pthread.h
+ */
+
+#ifndef QVI_GROUP_PTHREAD_H
+#define QVI_GROUP_PTHREAD_H
+
+#include "qvi-common.h"
+#include "qvi-group.h"
+#include "qvi-pthread.h"
+
+struct qvi_group_pthread_s : public qvi_group_s {
+    /** Underlying group instance. */
+    qvi_pthread_group_t *thgroup = nullptr;
+    /** Constructor. */
+    qvi_group_pthread_s(void) = default;
+    /** Destructor. */
+    virtual ~qvi_group_pthread_s(void)
+    {
+        qvi_delete(&thgroup);
+    }
+
+    virtual qvi_task_t *
+    task(void)
+    {
+        return thgroup->task();
+    }
+
+    virtual int
+    rank(void)
+    {
+        return thgroup->rank();
+    }
+
+    virtual int
+    size(void)
+    {
+        return thgroup->size();
+    }
+
+    virtual int
+    barrier(void)
+    {
+        return thgroup->barrier();
+    }
+
+    virtual int
+    make_intrinsic(
+        qv_scope_intrinsic_t
+    ) {
+        // Nothing to do.
+        return QV_SUCCESS;
+    }
+
+    virtual int
+    self(
+        qvi_group_s **child
+    );
+
+    virtual int
+    thsplit(
+        int,
+        qvi_group_s **
+    ) {
+        // TODO(skg) Need to test this.
+        return QV_ERR_NOT_SUPPORTED;
+    }
+
+    virtual int
+    split(
+        int color,
+        int key,
+        qvi_group_s **child
+    );
+
+    virtual int
+    gather(
+        qvi_bbuff_t *txbuff,
+        int root,
+        bool *shared,
+        qvi_bbuff_t ***rxbuffs
+    ) {
+        return thgroup->gather_bbuffs(
+           txbuff, root, shared, rxbuffs
+        );
+    }
+
+    virtual int
+    scatter(
+        qvi_bbuff_t **txbuffs,
+        int root,
+        qvi_bbuff_t **rxbuff
+    ) {
+        return thgroup->scatter_bbuffs(
+            txbuffs, root, rxbuff
+        );
+    }
+};
+typedef qvi_group_pthread_s qvi_group_pthread_t;
+
+struct qvi_zgroup_pthread_s : public qvi_group_pthread_s {
+    /** Default constructor. */
+    qvi_zgroup_pthread_s(void) = delete;
+    /** Constructor. */
+    qvi_zgroup_pthread_s(
+        int group_size
+    );
+};
+
+#endif
+
+/*
+ * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
+ */

src/qvi-group.cc

@@ -0,0 +1,38 @@
+/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2021-2024 Triad National Security, LLC
+ *                         All rights reserved.
+ *
+ * This file is part of the quo-vadis project. See the LICENSE file at the
+ * top-level directory of this distribution.
+ */
+
+/**
+ * @file qvi-group.cc
+ */
+
+#include "qvi-group.h"
+#include "qvi-group-pthread.h"
+#include "qvi-utils.h"
+
+int
+qvi_group_s::thsplit(
+    int nthreads,
+    qvi_group_s **child
+) {
+    qvi_group_pthread_t *ichild = nullptr;
+    int rc = qvi_new(&ichild);
+    if (rc != QV_SUCCESS) goto out;
+
+    rc = qvi_new(&ichild->thgroup, nthreads);
+out:
+    if (rc != QV_SUCCESS) {
+        qvi_delete(&ichild);
+    }
+    *child = ichild;
+    return rc;
+}
+
+/*
+ * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
+ */

src/qvi-group.h

@@ -27,8 +27,9 @@ using qvi_group_id_t = uint64_t;
 /**
  * Virtual base group class.
  */
-struct qvi_group_s {
+struct qvi_group_s : qvi_refc_s {
 protected:
+    // TODO(skg) Remove from base.
     /** Task associated with this group. */
     qvi_task_t *m_task = nullptr;
 public:
@@ -44,7 +45,7 @@ struct qvi_group_s {
         qvi_delete(&m_task);
     }
     /** Returns pointer to the caller's task information. */
-    qvi_task_t *
+    virtual qvi_task_t *
     task(void)
     {
         return m_task;
@@ -71,6 +72,14 @@ struct qvi_group_s {
     self(
         qvi_group_s **child
     ) = 0;
+    /**
+     * Creates a new thread group by splitting off of the caller's group.
+     */
+    virtual int
+    thsplit(
+        int nthreads,
+        qvi_group_s **child
+    );
     /**
      * Creates new groups by splitting this group based on color, key.
      * Returns the appropriate newly created child group to the caller.

src/qvi-pthread.cc

@@ -0,0 +1,21 @@
+/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2024      Triad National Security, LLC
+ *                         All rights reserved.
+ *
+ * This file is part of the quo-vadis project. See the LICENSE file at the
+ * top-level directory of this distribution.
+ */
+
+/**
+ * @file qvi-pthread.cc
+ */
+
+#include "qvi-pthread.h"
+#include "qvi-bbuff.h"
+#include "qvi-utils.h"
+#include "qvi-task.h"
+
+/*
+ * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
+ */