Merge pull request #1292 from cedardevs/1238-dashboards

1238 dashboards
cedardevs · Jul 6, 2020 · 9ca2302 · 9ca2302
2 parents a9ea8cc + ef5ad69
commit 9ca2302
Show file tree

Hide file tree

Showing 11 changed files with 678 additions and 240 deletions.
diff --git a/buildSrc/src/main/kotlin/utils.kt b/buildSrc/src/main/kotlin/utils.kt
@@ -41,7 +41,7 @@ object Versions {
     const val SNAKE_YAML = "1.24"
     const val REACTOR_BOM = "Dysprosium-SR7"
 
-    const val ONESTOP_SCHEMAS: String = "0.5.5"
+    const val ONESTOP_SCHEMAS: String = "0.6.0"
 }
 
 // data classes
@@ -87,4 +87,4 @@ fun parseDateISO(date: String): Date {
     val timeFormatter: DateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME
     val accessor: TemporalAccessor = timeFormatter.parse(date)
     return Date.from(Instant.from(accessor))
-}
+}
diff --git a/elastic-common/src/main/java/org/cedar/onestop/elastic/common/ElasticsearchConfig.java b/elastic-common/src/main/java/org/cedar/onestop/elastic/common/ElasticsearchConfig.java
@@ -119,12 +119,12 @@ public String jsonMapping(String alias) {
     return this.jsonMappings.getOrDefault(alias, null);
   }
 
-  public Map<String, ?> parsedMapping(String alias) {
+  public Map<String, Map> parsedMapping(String alias) {
     // retrieve JSON mapping for index alias
     return this.parsedMappings.getOrDefault(alias, Collections.emptyMap());
   }
 
-  public Map<String, ?> indexedProperties(String alias) {
+  public Map<String, Map> indexedProperties(String alias) {
     var parsed = (Map<String, Map>) parsedMapping(alias);
     var mappings = (Map<String, Map>) parsed.getOrDefault("mappings", Collections.emptyMap());
     return (Map<String, Map>) mappings.getOrDefault("properties", Collections.emptyMap());
@@ -168,4 +168,4 @@ public Boolean sitemapEnabled() {
     return SITEMAP_ENABLED;
   }
 
-}
+}
diff --git a/elastic-common/src/main/resources/dashboards/errors.ndjson b/elastic-common/src/main/resources/dashboards/errors.ndjson
diff --git a/elastic-common/src/main/resources/mappings/analysis_error_collectionIndex.json b/elastic-common/src/main/resources/mappings/analysis_error_collectionIndex.json
@@ -2,6 +2,10 @@
   "mappings": {
     "dynamic": "strict",
     "properties": {
+      "stagedDate": {
+        "type": "date",
+        "format": "epoch_millis"
+      },
       "dataAccess": {
         "properties": {
           "dataAccessExists": {
@@ -54,7 +58,7 @@
           "hierarchyLevelNameExists": {
             "type": "boolean"
           },
-          "matchesIdentifiers": {
+          "isGranule": {
             "type": "boolean"
           },
           "parentIdentifierExists": {
@@ -165,7 +169,7 @@
       "errors": {
         "type": "nested",
         "properties": {
-          "applicationSource": {
+          "source": {
             "type": "keyword"
           },
           "title": {
@@ -178,4 +182,4 @@
       }
     }
   }
-}
+}
diff --git a/elastic-common/src/main/resources/mappings/analysis_error_granuleIndex.json b/elastic-common/src/main/resources/mappings/analysis_error_granuleIndex.json
@@ -2,6 +2,13 @@
   "mappings": {
     "dynamic": "strict",
     "properties": {
+      "stagedDate": {
+        "type": "date",
+        "format": "epoch_millis"
+      },
+      "internalParentIdentifier": {
+        "type": "keyword"
+      },
       "dataAccess": {
         "properties": {
           "dataAccessExists": {
@@ -54,11 +61,14 @@
           "hierarchyLevelNameExists": {
             "type": "boolean"
           },
-          "matchesIdentifiers": {
+          "isGranule": {
             "type": "boolean"
           },
           "parentIdentifierExists": {
             "type": "boolean"
+          },
+          "parentIdentifierString": {
+            "type": "text"
           }
         }
       },
@@ -165,7 +175,7 @@
       "errors": {
         "type": "nested",
         "properties": {
-          "applicationSource": {
+          "source": {
             "type": "keyword"
           },
           "title": {
@@ -178,4 +188,4 @@
       }
     }
   }
-}
+}
diff --git a/indexer/src/main/java/org/cedar/onestop/indexer/util/IndexingInput.java b/indexer/src/main/java/org/cedar/onestop/indexer/util/IndexingInput.java
@@ -72,6 +72,10 @@ public Set<String> getTargetAnalysisAndErrorsIndexFields() {
     }
   }
 
+  public RecordType getRecordType() {
+    return recordType;
+  }
+
   @Override
   public String toString() {
     return "IndexingInput {" +

diff --git a/indexer/src/main/java/org/cedar/onestop/indexer/util/IndexingUtils.java b/indexer/src/main/java/org/cedar/onestop/indexer/util/IndexingUtils.java
@@ -76,7 +76,8 @@ public static DocWriteRequest<?> buildAnalysisAndErrorWriteRequest(String indexN
     }
     else {
       var formattedRecord = new HashMap<String, Object>();
-      formattedRecord.putAll(TransformationUtils.reformatMessageForAnalysisAndErrors(input.getValue().value(), input.getTargetAnalysisAndErrorsIndexFields()));
+      formattedRecord.putAll(TransformationUtils.reformatMessageForAnalysis(input.getValue().value(), input.getTargetAnalysisAndErrorsIndexFields(), input.getRecordType()));
+      formattedRecord.put("stagedDate", input.getValue().timestamp());
       return new IndexRequest(indexName).opType(opType).id(input.getKey()).source(formattedRecord);
     }
   }