gchq · GCHQDeveloper314 · Feb 14, 2024 · Jan 12, 2024 · Jan 12, 2024 · Jan 12, 2024
diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,10 +27,14 @@ jobs:
     steps:
 
     - name: Maximize build space
-      uses: easimon/maximize-build-space@v4
+      uses: easimon/maximize-build-space@v10
       with:
         remove-dotnet: 'true'
         remove-android: 'true'
+        remove-haskell: 'true'
+        remove-codeql: 'true'
+        remove-docker-images: 'true'
+        swap-size-mb: 1024
         overprovision-lvm: 'true'
 
     - uses: actions/checkout@v4

diff --git a/docker/accumulo1.env b/docker/accumulo1.env
@@ -3,7 +3,7 @@ HADOOP_VERSION=3.2.2
 ACCUMULO_VERSION=1.9.3
 HADOOP_CONF_DIR=/etc/hadoop/conf
 ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
 SPARK_VERSION=3.1.2
diff --git a/docker/accumulo2.env b/docker/accumulo2.env
@@ -3,7 +3,7 @@ HADOOP_VERSION=3.3.3
 ACCUMULO_VERSION=2.0.1
 HADOOP_CONF_DIR=/etc/hadoop/conf
 ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
 SPARK_VERSION=3.1.2
diff --git a/docker/gaffer-federated-integration-tests/.env b/docker/gaffer-federated-integration-tests/.env
@@ -1,6 +1,6 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

diff --git a/docker/gaffer-integration-tests/.env b/docker/gaffer-integration-tests/.env
@@ -1,6 +1,6 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

diff --git a/docker/gaffer-kerberos/.env b/docker/gaffer-kerberos/.env
@@ -1,4 +1,4 @@
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 GAFFER_KRB_PASSWORD=change-password-for-gaffer
 
 ZOOKEEPER_VERSION=3.7.1

diff --git a/docker/gaffer-kerberos/gaffer-integration-tests-krb/.env b/docker/gaffer-kerberos/gaffer-integration-tests-krb/.env
@@ -1,6 +1,6 @@
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 GAFFER_KRB_PASSWORD=change-password-for-gaffer
-GAFFER_TESTER_VERSION=2.0.0
+GAFFER_TESTER_VERSION=2.1.0
 
 ZOOKEEPER_VERSION=3.7.1
 ZOOKEEPER_KRB_PASSWORD=change-password-for-zookeeper

diff --git a/docker/gaffer-kerberos/gaffer-krb/Dockerfile b/docker/gaffer-kerberos/gaffer-krb/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE_NAME=gchq/gaffer
-ARG BASE_IMAGE_TAG=2.0.0
+ARG BASE_IMAGE_TAG=2.1.0
 
 ARG USER=accumulo
 ARG GROUP=accumulo

diff --git a/docker/gaffer-kerberos/gaffer-rest-krb/Dockerfile b/docker/gaffer-kerberos/gaffer-rest-krb/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE_NAME=gchq/gaffer-rest
-ARG BASE_IMAGE_TAG=2.0.0
+ARG BASE_IMAGE_TAG=2.1.0
 
 FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
 

diff --git a/docker/gaffer-pyspark-notebook/.env b/docker/gaffer-pyspark-notebook/.env
@@ -1,6 +1,6 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 SPARK_VERSION=3.1.2

diff --git a/docker/gaffer-pyspark-notebook/Dockerfile b/docker/gaffer-pyspark-notebook/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -60,7 +60,7 @@ ARG KUBECTL_VERSION=1.23.0
 RUN curl -fLo /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
 	chmod +x /bin/kubectl
 
-ARG GAFFERPY_VERSION=gafferpy-2.0.0
+ARG GAFFERPY_VERSION=gafferpy-2.1.0
 RUN git clone -b ${GAFFERPY_VERSION} --depth 1 https://github.com/gchq/gafferpy && \
 	pushd gafferpy && \
 	python setup.py install && \

diff --git a/docker/gaffer-rest/.env b/docker/gaffer-rest/.env
@@ -1,2 +1,2 @@
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
diff --git a/docker/gaffer-rest/Dockerfile b/docker/gaffer-rest/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ ARG BASE_IMAGE_TAG=8-jre-alpine
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
 ARG ACCUMULO_VERSION=2.0.1
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 

diff --git a/docker/gaffer-road-traffic-loader/.env b/docker/gaffer-road-traffic-loader/.env
@@ -1,5 +1,5 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

diff --git a/docker/gaffer-road-traffic-loader/Dockerfile b/docker/gaffer-road-traffic-loader/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=8u322-jre-slim-bullseye
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 

diff --git a/docker/gaffer-road-traffic-loader/config/schema/types.json b/docker/gaffer-road-traffic-loader/config/schema/types.json
@@ -100,12 +100,12 @@
       }
     },
     "hllp": {
-      "class": "com.clearspring.analytics.stream.cardinality.HyperLogLogPlus",
+      "class": "org.apache.datasketches.hll.HllSketch",
       "aggregateFunction": {
-        "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.binaryoperator.HyperLogLogPlusAggregator"
+        "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator"
       },
       "serialiser": {
-        "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.serialisation.HyperLogLogPlusSerialiser"
+        "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser"
       }
     },
     "counts.freqmap": {

diff --git a/docker/gaffer/.env b/docker/gaffer/.env
@@ -1,5 +1,5 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

diff --git a/docker/gaffer/Dockerfile b/docker/gaffer/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ ARG BASE_IMAGE_TAG=2.0.1
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
 ARG BASE_IMAGE_TAG
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_LIBS=bitmap-library,sketches-library,time-library
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
@@ -60,5 +60,9 @@ RUN	allFilesDownloaded="TRUE" && \
 		done \
 	fi
 
+# Hotfix for Gaffer 2.1.0, see Gaffer issue #3159
+RUN wget -nv "${GAFFER_DOWNLOAD_URL}/org/apache/datasketches/datasketches-memory/2.2.0/datasketches-memory-2.2.0.jar"
+RUN wget -nv "${GAFFER_DOWNLOAD_URL}/org/apache/datasketches/datasketches-java/4.0.0/datasketches-java-4.0.0.jar"
+
 FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
 COPY --from=builder --chown=root:root /jars/*.jar /opt/accumulo/lib/ext/
diff --git a/docker/gremlin-gaffer/.env b/docker/gremlin-gaffer/.env
@@ -1,5 +1,5 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 GREMLIN_VERSION=3.6.4
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3

diff --git a/docker/gremlin-gaffer/Dockerfile b/docker/gremlin-gaffer/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ ARG BASE_IMAGE_TAG=3.6.4
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
 ARG BASE_IMAGE_TAG
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 

diff --git a/kubernetes/accumulo/Chart.yaml b/kubernetes/accumulo/Chart.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@ apiVersion: v2
 name: accumulo
 description: A Key-Value Accumulo store
 type: application
-version: 2.0.0 # managed version
+version: 2.1.0 # managed version
 appVersion: 2.0.1
 home: https://github.com/gchq/Gaffer
 sources:
@@ -26,6 +26,6 @@ dependencies:
     repository: https://charts.helm.sh/incubator/
     condition: zookeeper.enabled
   - name: hdfs
-    version: ^2.0.0 # managed version
+    version: ^2.1.0 # managed version
     repository: file://../hdfs/
     condition: hdfs.enabled
diff --git a/kubernetes/gaffer-jhub/Chart.yaml b/kubernetes/gaffer-jhub/Chart.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@ apiVersion: v2
 name: gaffer-jhub
 description: A Jupyter Hub instance, with Gaffer integrations
 type: application
-version: 2.0.0 # managed version
+version: 2.1.0 # managed version
 home: https://github.com/gchq/Gaffer
 sources:
   - https://github.com/gchq/gaffer-docker

diff --git a/kubernetes/gaffer-jhub/values.yaml b/kubernetes/gaffer-jhub/values.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,7 +65,7 @@ jupyterhub:
         kubespawner_override:
           image: jupyter/minimal-notebook:399cbb986c6b
       - display_name: "Gaffer pySpark Notebook"
-        description: "Python 3, Hadoop 3.3.3, Spark 3.1.2, AWS CLI 2, kubectl 1.23.0, gafferpy 2.0.0" # managed
+        description: "Python 3, Hadoop 3.3.3, Spark 3.1.2, AWS CLI 2, kubectl 1.23.0, gafferpy 2.1.0" # managed
         slug: "gaffer-pyspark"
         default: true
         enable_hdfs: true
@@ -74,7 +74,7 @@ jupyterhub:
         spark_image: gchq/spark-py:3.1.2 # managed
         spark_ingress_host: "{{USERNAME}}-{{SERVERNAME}}.spark.example.com"
         kubespawner_override:
-          image: gchq/gaffer-pyspark-notebook:2.0.0 # managed
+          image: gchq/gaffer-pyspark-notebook:2.1.0 # managed
 optionsServer:
   replicaCount: 1
   image:
@@ -141,4 +141,4 @@ optionsServer:
 testImages:
   python:
     repository: gchq/gaffer-pyspark-notebook
-    tag: 2.0.0 # managed version
+    tag: 2.1.0 # managed version
diff --git a/kubernetes/gaffer-road-traffic/Chart.yaml b/kubernetes/gaffer-road-traffic/Chart.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@ apiVersion: v2
 name: gaffer-road-traffic
 description: A Gaffer instance containing sample GB road traffic data from the Department of Transport
 type: application
-version: 2.0.0 # managed version
-appVersion: 2.0.0 # managed version
+version: 2.1.0 # managed version
+appVersion: 2.1.0 # managed version
 home: https://github.com/gchq/Gaffer
 sources:
   - https://github.com/gchq/gaffer-docker
   - https://data.gov.uk/dataset/208c0e7b-353f-4e2d-8b7a-1a7118467acc/gb-road-traffic-counts
 dependencies:
   - name: gaffer
-    version: ^2.0.0 # managed version
+    version: ^2.1.0 # managed version
     repository: file://../gaffer/
diff --git a/kubernetes/gaffer-road-traffic/values.yaml b/kubernetes/gaffer-road-traffic/values.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ labels: {}
 loader:
   image:
     repository: gchq/gaffer-road-traffic-loader
-    tag: 2.0.0 # managed version
+    tag: 2.1.0 # managed version
     pullPolicy: IfNotPresent
     pullSecrets: []
   resources: {}
@@ -206,12 +206,12 @@ gaffer:
               }
             },
             "hllp": {
-              "class": "com.clearspring.analytics.stream.cardinality.HyperLogLogPlus",
+              "class": "org.apache.datasketches.hll.HllSketch",
               "aggregateFunction": {
-                "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.binaryoperator.HyperLogLogPlusAggregator"
+                "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator"
               },
               "serialiser": {
-                "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.serialisation.HyperLogLogPlusSerialiser"
+                "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser"
               }
             },
             "counts.freqmap": {

diff --git a/kubernetes/gaffer/Chart.yaml b/kubernetes/gaffer/Chart.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@ apiVersion: v2
 name: gaffer
 description: A large-scale entity and relation database supporting aggregation of properties
 type: application
-version: 2.0.0 # managed version
-appVersion: 2.0.0
+version: 2.1.0 # managed version
+appVersion: 2.1.0
 home: https://github.com/gchq/Gaffer
 sources:
   - https://github.com/gchq/gaffer-docker
 dependencies:
   - name: accumulo
-    version: ^2.0.0 # managed version
+    version: ^2.1.0 # managed version
     repository: file://../accumulo/
     condition: accumulo.enabled