Resource Integration

examples/CMakeLists.txt

@@ -28,7 +28,22 @@ if (CHAI_ENABLE_CUDA OR CHAI_ENABLE_HIP)
     NAME chai-example.exe
     SOURCES example.cpp
     DEPENDS_ON ${chai_umpire_example_depends})
-
+  blt_add_executable(
+    NAME resource-depends.exe
+    SOURCES resource-depends.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
+  blt_add_executable(
+    NAME resource-simple.exe
+    SOURCES resource-simple.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
+  blt_add_executable(
+    NAME resource-not-managed.exe
+    SOURCES resource-not-managed.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
+  blt_add_executable(
+    NAME resource-multi-array.exe
+    SOURCES resource-multi-array.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
   blt_add_executable(
     NAME pinned.exe
     SOURCES pinned.cpp

examples/resource-depends.cpp

@@ -0,0 +1,52 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "camp/resource.hpp"
+#include "chai/ManagedArray.hpp"
+
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+  constexpr std::size_t ARRAY_SIZE{1000};
+  int clockrate{get_clockrate()}; 
+
+  camp::resources::Resource dev1{camp::resources::Cuda{}};
+  camp::resources::Resource dev2{camp::resources::Cuda{}};
+  camp::resources::Resource host{camp::resources::Host{}};
+
+  chai::ManagedArray<double> array1(ARRAY_SIZE);
+  chai::ManagedArray<double> array2(ARRAY_SIZE);
+
+  auto e1 = forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      array1[i] = i;
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  auto e2 = forall(&dev2, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      array2[i] = -1;
+      gpu_time_wait_for(20, clockrate);
+  });
+
+  dev1.wait_for(&e2);
+
+  forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      array1[i] *= array2[i];
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  array1.move(chai::CPU, &dev1);
+
+  forall(&host, 0, 10, [=] CHAI_HOST_DEVICE (int i) {
+      printf("%f ", array1[i]);
+  });
+  printf("\n");
+}

examples/resource-multi-array.cpp

@@ -0,0 +1,50 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "camp/resource.hpp"
+#include "chai/ManagedArray.hpp"
+
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+  constexpr std::size_t ARRAY_SIZE{1000};
+  int clockrate{get_clockrate()}; 
+
+  chai::ManagedArray<double> array1(ARRAY_SIZE);
+
+  camp::resources::Resource dev1{camp::resources::Cuda{}};
+  camp::resources::Resource dev2{camp::resources::Cuda{}};
+
+
+  auto e1 = forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      if (i % 2 == 0) {
+        array1[i] = i;
+        gpu_time_wait_for(10, clockrate);
+      }
+  });
+
+  auto e2 = forall(&dev2, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      if (i % 2 == 1) {
+        gpu_time_wait_for(20, clockrate);
+        array1[i] = i;
+      }
+  });
+
+  array1.move(chai::CPU, &dev1);
+
+  camp::resources::Resource host{camp::resources::Host{}};
+
+  forall(&host, 0, 10, [=] CHAI_HOST_DEVICE (int i) {
+      printf("%f ", array1[i]);
+  });
+  printf("\n");
+}

examples/resource-not-managed.cpp

@@ -0,0 +1,52 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "camp/resource.hpp"
+
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+  constexpr std::size_t ARRAY_SIZE{1000};
+  int clockrate{get_clockrate()};
+
+  camp::resources::Resource dev1{camp::resources::Cuda{}};
+  camp::resources::Resource dev2{camp::resources::Cuda{}};
+  camp::resources::Resource host{camp::resources::Host{}};
+
+  float * d_array1 = dev1.allocate<float>(1000);
+  float * d_array2 = dev2.allocate<float>(1000);
+  float * h_array1 = host.allocate<float>(1000);
+
+  auto e1 = forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      d_array1[i] = i;
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  auto e2 = forall(&dev2, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      d_array2[i] = -1;
+      gpu_time_wait_for(20, clockrate);
+  });
+
+  dev1.wait_for(&e2);
+
+  forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      d_array1[i] *= d_array2[i];
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  dev1.memcpy(h_array1, d_array1, sizeof(float) * 1000);
+
+  forall(&host, 0, 10, [=] CHAI_HOST_DEVICE (int i) {
+      printf("%f ", h_array1[i]);
+  });
+  printf("\n");
+}

examples/resource-simple.cpp

@@ -0,0 +1,57 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include "chai/ManagedArray.hpp"
+#include "camp/resource.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+
+  constexpr int NUM_ARRAYS = 16;
+  constexpr std::size_t ARRAY_SIZE{100};
+
+  std::vector<chai::ManagedArray<double>> arrays;
+  camp::resources::Resource host{camp::resources::Host{}}; 
+
+
+  int clockrate{get_clockrate()};
+
+  for (std::size_t i = 0; i < NUM_ARRAYS; ++i) {
+    arrays.push_back(chai::ManagedArray<double>(ARRAY_SIZE));
+  }
+
+  for (auto array : arrays) {
+    forall(&host, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+        array[i] = i;
+    }); 
+  }
+
+  for (auto array : arrays) {
+    camp::resources::Resource resource{camp::resources::Cuda{}}; 
+
+    forall(&resource, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+        array[i] = array[i] * 2.0;
+        gpu_time_wait_for(20, clockrate);
+    });
+
+    array.move(chai::CPU, &resource);
+  }
+
+  for (auto array : arrays) {
+    forall(&host, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+        if (i == 25) {
+          printf("array[%d] = %f \n", i, array[i]);
+        }
+    }); 
+  }
+}

scripts/spack_packages/chai/package.py

@@ -74,9 +74,12 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage):
             multi=False, description='Tests to run')
 
     depends_on('umpire')
-    depends_on('raja', when="+raja")
-
     depends_on('umpire@main', when='@main')
+
+    depends_on('camp')
+    depends_on('camp@main', when='@main')
+
+    depends_on('raja', when="+raja")
     depends_on('raja@main', when="@main+raja")
 
     depends_on('cmake@3.14:', type='build')

src/chai/ActiveResourceManager.hpp

@@ -0,0 +1,90 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-23, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#ifndef CHAI_ActiveResourceManager_HPP
+#define CHAI_ActiveResourceManager_HPP
+
+#include "camp/resource.hpp"
+
+#include <array>
+#include <vector>
+
+namespace chai
+{
+
+/*!
+ * \Class to store list of Resource pointers. Holds data on the stack 
+ * until a certain threshold, then uses heap memory.
+ */
+class ActiveResourceManager {
+
+  /*!
+   * Size of array on the stack.
+   */
+  static constexpr int BASE_SIZE = 16;
+
+  /*!
+   * Base array on the stack.
+   */
+  std::array<camp::resources::Resource*, BASE_SIZE> m_res_base;
+
+  /*!
+   * Heap container for extra resources if more than BASE_SIZE pushed.
+   */
+  std::vector<camp::resources::Resource*> m_res_overflow;
+
+  /*!
+   * Current number of active resources in the list.
+   */
+  int m_size = 0;
+
+public:
+  /*!
+   * Default constructor.
+   */
+  ActiveResourceManager() = default;
+
+  /*!
+   * Return current size of the resource list.
+   *
+   * \return The current size of the resource list.
+   */
+  int size();
+
+  /*!
+   * Push a new resource onto the list. 
+   *
+   * \param res The resource to add.
+   */
+  void push_back(camp::resources::Resource* res);
+
+  /*!
+   * Clear all values on the heap and set m_size to 0. 
+   */
+  void clear();
+
+  /*!
+   * Check if empty.
+   *
+   * \return Whether or not the resource manager is empty.
+   */
+  bool is_empty() const;
+
+  /*!
+   * Get resource at given index. 
+   *
+   * \param i The index at which to get a resource.
+   *
+   * \return The resource at the given index.
+   */
+  camp::resources::Resource* operator [](int i) const;
+};
+
+}  // end of namespace chai
+
+#include "chai/ActiveResourceManager.inl"
+
+#endif  // CHAI_ActiveResourceManager_HPP