Merge pull request #5540 from Artemy-Mellanox/topic/fix_ro_key-1.9

UCT/IB: Fix create STRICT_ORDER key - v1.9.x
openucx · Aug 7, 2020 · 3a8c2fa · 3a8c2fa
2 parents 3059eb4 + 21f0b2f
commit 3a8c2fa
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 8 deletions.
diff --git a/src/uct/ib/base/ib_md.c b/src/uct/ib/base/ib_md.c
@@ -885,9 +885,9 @@ static ucs_status_t uct_ib_mkey_pack(uct_md_h uct_md, uct_mem_h uct_memh,
     /* create umr only if a user requested atomic access to the
      * memory region and the hardware supports it.
      */
-    if ((((memh->flags & UCT_IB_MEM_ACCESS_REMOTE_ATOMIC) &&
-        !(memh->flags & UCT_IB_MEM_FLAG_ATOMIC_MR)) ||
-        (memh->flags & UCT_IB_MEM_FLAG_RELAXED_ORDERING)) &&
+    if (((memh->flags & UCT_IB_MEM_ACCESS_REMOTE_ATOMIC) ||
+         (memh->flags & UCT_IB_MEM_FLAG_RELAXED_ORDERING)) &&
+        !(memh->flags & UCT_IB_MEM_FLAG_ATOMIC_MR) &&
         (memh != md->global_odp))
     {
         /* create UMR on-demand */

diff --git a/test/gtest/uct/ib/test_ib_md.cc b/test/gtest/uct/ib/test_ib_md.cc
@@ -75,6 +75,9 @@ void test_ib_md::ib_md_umr_check(void *rkey_buffer,
     status = uct_md_mkey_pack(md(), memh, rkey_buffer);
     EXPECT_UCS_OK(status);
 
+    status = uct_md_mkey_pack(md(), memh, rkey_buffer);
+    EXPECT_UCS_OK(status);
+
 #ifdef HAVE_MLX5_HW
     uct_ib_md_t *ib_md = (uct_ib_md_t *)md();
 
@@ -111,17 +114,22 @@ bool test_ib_md::has_ksm() const {
 }
 
 bool test_ib_md::check_umr(uct_ib_md_t *ib_md) const {
+    if (ib_md->dev.flags & UCT_IB_DEVICE_FLAG_MLX5_PRM) {
 #if HAVE_DEVX
-    return has_ksm();
+        if (ucs_derived_of(ib_md, uct_ib_mlx5_md_t)->flags & UCT_IB_MLX5_MD_FLAG_DEVX) {
+            return has_ksm();
+        } else {
+            return ib_md->relaxed_order;
+        }
 #elif HAVE_EXP_UMR
-    if (ib_md->dev.flags & UCT_IB_DEVICE_FLAG_MLX5_PRM) {
         uct_ib_mlx5_md_t *mlx5_md = ucs_derived_of(ib_md, uct_ib_mlx5_md_t);
         return mlx5_md->umr_qp != NULL;
-    }
-    return false;
 #else
-    return false;
+        return false;
 #endif
+    } else {
+        return ib_md->relaxed_order;
+    }
 }
 
 UCS_TEST_P(test_ib_md, ib_md_umr_rcache, "REG_METHODS=rcache") {
@@ -150,6 +158,12 @@ UCS_TEST_P(test_ib_md, ib_md_umr_ksm) {
     ib_md_umr_check(&rkey_buffer[0], has_ksm(), UCT_IB_MD_MAX_MR_SIZE + 0x1000);
 }
 
+UCS_TEST_P(test_ib_md, relaxed_order, "PCI_RELAXED_ORDERING=on") {
+    std::string rkey_buffer(md_attr().rkey_packed_size, '\0');
+
+    ib_md_umr_check(&rkey_buffer[0], true);
+}
+
 #if HAVE_UMR_KSM
 UCS_TEST_P(test_ib_md, umr_noninline_klm, "MAX_INLINE_KLM_LIST=1") {