Gh-334: Smoother gaffer gremlin deployment

.github/workflows/continuous-integration.yaml

@@ -55,7 +55,7 @@ jobs:
 
     - name: Deploy to Kubernetes
       run: ./cd/deploy_to_kind.sh
-  
+
     - name: Run gaffer-road-traffic Tests
       run: helm test gaffer || (kubectl get po && kubectl describe po && kubectl logs -l app.kubernetes.io/component=test --tail=-1 && df -h && false)
 

cd/build_images.sh

@@ -18,31 +18,36 @@
 # Required for variables from sourced env file to automatically be visible to docker compose.
 set -e -a
 
-root_directory="$( cd $(dirname $(dirname $0)) > /dev/null 2>&1 && pwd )"
-cd $root_directory
+ROOT_DIR="$(readlink -f "$(dirname "$(dirname "${0}")")")"
 
-if [ ! -z "$1" ]; then
-	ENV_FILE=$1
-else
-	echo "Error - Environment file not set"; exit 1;
-fi
+pushd "${ROOT_DIR}" || exit 1
 
-# The following command sets:
+# The following env file will be sourced to set:
 # HADOOP_VERSION
 # GAFFER_VERSION
 # GAFFERPY_VERSION
 # ACCUMULO_VERSION
 # SPARK_VERSION
-source "${ENV_FILE}"
+# TINKERPOP_VERSION
+if [[ -f "${1}" ]]; then
+    source "${1}"
+else
+    echo "Error - Environment file not set"
+    exit 1
+fi
 
 # Builds all of the Gaffer and Accumulo related images:
 docker compose --project-directory ./docker/accumulo/ -f ./docker/accumulo/docker-compose.yaml build
 docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./docker/gaffer-road-traffic-loader/docker-compose.yaml build
 # Builds all of the notebook related images:
 docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook
 docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build
+# Builds the Gaffer Gremlin server
+./docker/gaffer-gremlin/build.sh
 
 # Set $JHUB_OPTIONS_SERVER_VERSION
 source ./docker/gaffer-jhub-options-server/get-version.sh
 # Builds the jhub options server:
 docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build
+
+popd || exit 1

cd/publish_images.sh

@@ -101,6 +101,7 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}"
+pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}"
 pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}"
 pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}"
 pushContainer gchq/spark-py "${SPARK_VERSION}"

docker/accumulo2.env

@@ -3,7 +3,8 @@ HADOOP_VERSION=3.3.3
 ACCUMULO_VERSION=2.0.1
 HADOOP_CONF_DIR=/etc/hadoop/conf
 ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
 SPARK_VERSION=3.1.2
+TINKERPOP_VERSION=3.7.1

docker/gaffer-gremlin/Dockerfile

@@ -0,0 +1,27 @@
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
+ARG BASE_IMAGE_TAG=3.7.1
+
+FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
+
+# Copy JARS
+COPY ./target/dependency/*.jar ext/gafferpop/plugin/
+
+# Copy configs
+COPY ./conf/ conf/
+
+# Set server to run
+CMD ["conf/gaffer-gremlin-server.yaml"]

docker/gaffer-gremlin/README.md

@@ -0,0 +1,38 @@
+# gaffer-gremlin
+
+## Build
+
+Run the supplied `build.sh` script with the required environment vars to pull
+all dependencies and build the container.
+
+_Note requires Maven and Docker_
+
+Optionally run each command separately to configure the containers tags etc.
+
+## Configure
+
+The container will use the Gaffer Proxy store to connect to an existing graph
+and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)).
+
+The Gaffer graph the container will connect to can be configured as usual by
+editing the `store.properties` file, which you can also bind mount over on an
+existing image.
+
+The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml`
+this again can be modified as needed or bind mounted over. Please see the
+[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/)
+for more information on configuring this image.
+
+## Run
+
+Simply run the container to publish the Gremlin server this can then be
+connected to via the address specified in the server's yaml config to use
+Gremlin traversal. The server can then be connected via the Gremlin
+console e.g. if using python
+
+```python
+from gremlin_python.process.anonymous_traversal_source import traversal
+
+g = traversal().withRemote(
+    DriverRemoteConnection('ws://localhost:8182/gremlin'))
+```

docker/gaffer-gremlin/build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")"
+
+# Check if already have env var for version
+[[ -z "${GAFFER_VERSION}" ]] && \
+    echo "Missing GAFFER_VERSION env var" && \
+    exit 1
+[[ -z "${TINKERPOP_VERSION}" ]] && \
+    echo "Missing TINKERPOP_VERSION env var" && \
+    exit 1
+
+# Build from relevant directory
+pushd "${SCRIPT_DIR}" || exit 1
+    # Download JARs
+    mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}"
+
+    # Build container
+    docker build -t gaffer-gremlin:"${GAFFER_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" .
+
+    # Clean
+    mvn clean
+popd || exit 1
+
+echo "Build Successful"

docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml

@@ -0,0 +1,45 @@
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+host: localhost
+port: 8182
+evaluationTimeout: 30000
+graphs: {
+  graph: conf/gafferpop/gafferpop.properties
+}
+scriptEngines: {
+  gremlin-groovy: {
+    plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
+               uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {},
+               org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true},
+               org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
+               org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}
+    }
+  }
+}
+serializers:
+  - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
+
+metrics: {
+  slf4jReporter: {enabled: true, interval: 180000}}
+strictTransactionManagement: false
+idleConnectionTimeout: 0
+keepAliveInterval: 0
+maxInitialLineLength: 4096
+maxHeaderSize: 8192
+maxChunkSize: 8192
+maxContentLength: 10485760
+maxAccumulationBufferComponents: 1024
+resultIterationBatchSize: 64
+enableAuditLog: true

docker/gaffer-gremlin/conf/gaffer/store.properties

@@ -0,0 +1,20 @@
+#
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore
+# These can be configured to an existing graph deployment
+gaffer.host=localhost
+gaffer.port=8080
+gaffer.context-root=/rest/latest

docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties

@@ -0,0 +1,19 @@
+#
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
+gaffer.graphId=graphProxy
+gaffer.storeproperties=conf/gaffer/store.properties
+gaffer.userId=user01

docker/gaffer-gremlin/pom.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2023 Crown Copyright
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <!-- Utility pom just used to down load Gaffer JARS and dependencies -->
+    <groupId>uk.gov.gchq.gaffer.docker</groupId>
+    <artifactId>gaffer-gremlin</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <!-- Properties can be overriden for CI/CD -->
+    <properties>
+        <gaffer.version>2.1.0</gaffer.version>
+        <tinkerpop.version>3.7.1</tinkerpop.version>
+        <jersey.version>2.36</jersey.version>
+    </properties>
+
+    <dependencies>
+        <!-- Gaffer dependencies -->
+        <dependency>
+            <groupId>uk.gov.gchq.gaffer</groupId>
+            <artifactId>tinkerpop</artifactId>
+            <version>${gaffer.version}</version>
+        </dependency>
+        <!-- Proxy store needs this -->
+        <dependency>
+            <groupId>org.glassfish.jersey.inject</groupId>
+            <artifactId>jersey-hk2</artifactId>
+            <version>${jersey.version}</version>
+        </dependency>
+        <!-- TinkerPop dependencies -->
+        <dependency>
+            <groupId>org.apache.tinkerpop</groupId>
+            <artifactId>gremlin-core</artifactId>
+            <version>${tinkerpop.version}</version>
+        </dependency>
+    </dependencies>
+
+</project>