Merge pull request #530 from FgForrest/dev

Base image, logging and stability fixes

.github/workflows/docker-latest.yml

@@ -53,7 +53,8 @@ jobs:
         id: major_version
         # create a new output variable with the major version (like 2024.4)
         run: |        
-          echo "$(cat version.txt)"
+          echo "fullVersion=$(cat version.txt)"
+          echo "fullVersion=$(cat version.txt)" >> $GITHUB_ENV
           echo "version=$(cat version.txt | cut -d'.' -f1,2)"
           echo "version=$(cat version.txt | cut -d'.' -f1,2)" >> $GITHUB_ENV    
 
@@ -71,6 +72,8 @@ jobs:
           tags: |
             ${{ vars.CI_REGISTRY_USER }}/${{ env.RELEASE_IMAGE }}
             ${{ vars.CI_REGISTRY_USER }}/evitadb:${{ env.version }}
+            ${{ vars.CI_REGISTRY_USER }}/evitadb:${{ env.fullVersion }}
           platforms: linux/amd64,linux/arm64/v8
           build-args: |
             EVITA_JAR_NAME=${{ env.EVITA_JAR_NAME }}
+            VERSION=${{ env.fullVersion }}

docker/Dockerfile

@@ -1,15 +1,27 @@
-FROM index.docker.io/azul/zulu-openjdk-alpine:17-jre-latest
+FROM index.docker.io/azul/zulu-openjdk-alpine:17-latest
 
 ## not supposed to be overriden in container
-ENV EVITA_HOME="/evita"
 ARG EVITA_JAR_NAME
+ARG VERSION=not_set
+ENV EVITA_HOME="/evita"
 ENV EVITA_JAR_NAME="$EVITA_JAR_NAME"
 
+# Labels with dynamic information based on environment variables and current date
+LABEL vendor="FG Forrest, a.s." \
+      io.evitadb.version="${VERSION}" \
+      io.evitadb.release-date="$(date +%Y-%m-%d)"
+
 USER root
 
+# Install bash
+RUN apk add --no-cache bash
+
+# Create necessary folders
 RUN set -ex \
     && mkdir "$EVITA_HOME" "$EVITA_HOME/bin" "$EVITA_HOME/conf" "$EVITA_HOME/data" "$EVITA_HOME/certificates" \
     && : ## end
+
+# Copy files
 COPY "entrypoint.sh" "/"
 COPY "$EVITA_JAR_NAME" "$EVITA_HOME/bin"
 COPY "evita-configuration.yaml" "$EVITA_HOME/conf"

docker/evita-configuration.yaml

@@ -18,6 +18,8 @@ storage:
   outputBufferSize: ${storage.outputBufferSize:4MB}
   maxOpenedReadHandles: ${storage.maxOpenedReadHandles:12}
   computeCRC32C: ${storage.computeCRC32C:true}
+  minimalActiveRecordShare: ${storage.minimalActiveRecordShare:0.5}
+  fileSizeCompactionThresholdBytes: ${storage.fileSizeCompactionThresholdBytes:100M}
 
 transaction:
   transactionWorkDirectory: ${transaction.workDirectory:null}

evita_server/src/main/java/io/evitadb/server/log/AppLogJsonLayout.java

@@ -23,6 +23,8 @@
 
 package io.evitadb.server.log;
 
+import ch.qos.logback.classic.pattern.MessageConverter;
+import ch.qos.logback.classic.pattern.ThrowableProxyConverter;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.CoreConstants;
 import ch.qos.logback.core.LayoutBase;
@@ -46,9 +48,18 @@ public class AppLogJsonLayout extends LayoutBase<ILoggingEvent> {
 	private static final String[] REPLACEMENTS_FOR_ESCAPED_CHARS = new String[] { "\\r\\n", "\\n", "\\r", "\\f", "\\b", "\\\\", "\\\"" };
 
 	private final CachingDateFormatter cachingDateFormatter = new CachingDateFormatter("yyyy-MM-dd'T'HH:mm:ss.SSSZ", null);
+	private final MessageConverter messageConverter = new MessageConverter();
+	private final ThrowableProxyConverter throwableProxyConverter = new ThrowableProxyConverter();
 
 	@Setter private boolean logTimestamp = true;
 
+	@Override
+	public void start() {
+		messageConverter.start();
+		throwableProxyConverter.start();
+		super.start();
+	}
+
 	@Override
 	public String doLayout(ILoggingEvent event) {
 		final StringBuilder buf = new StringBuilder(512);
@@ -70,7 +81,13 @@ public String doLayout(ILoggingEvent event) {
 		buf.append(",");
 
 		buf.append("\"message\":\"");
-		buf.append(escapeMessage(event.getFormattedMessage()));
+		buf.append(escapeMessage(messageConverter.convert(event)));
+		buf.append("\"");
+
+		buf.append(",");
+
+		buf.append("\"throwableProxy\":\"");
+		buf.append(escapeMessage(throwableProxyConverter.convert(event)));
 		buf.append("\"");
 
 		buf.append(",");

evita_store/evita_store_key_value/src/main/java/io/evitadb/store/kryo/ObservableOutputKeeper.java

@@ -203,13 +203,13 @@ private OpenedOutputToFile getOpenedOutputToFile(
 	 * This method also re-plans the next cache cut if the cache is not empty.
 	 */
 	private long cutOutputCache() {
-		final long now = System.currentTimeMillis();
+		final long threshold = System.currentTimeMillis() - CUT_OUTPUTS_AFTER_INACTIVITY_MS;
 		long oldestNotCutEntryTouchTime = -1L;
 		final Iterator<OpenedOutputToFile> iterator = cachedOutputToFiles.values().iterator();
 		while (iterator.hasNext()) {
 			final OpenedOutputToFile outputToFile = iterator.next();
 			final boolean oldestRecord = oldestNotCutEntryTouchTime == -1L || outputToFile.getLastReadTime() < oldestNotCutEntryTouchTime;
-			if (outputToFile.getLastReadTime() - CUT_OUTPUTS_AFTER_INACTIVITY_MS > now) {
+			if (outputToFile.getLastReadTime() < threshold) {
 				if (outputToFile.isLeased()) {
 					oldestNotCutEntryTouchTime = outputToFile.getLastReadTime();
 				} else {
@@ -221,7 +221,7 @@ private long cutOutputCache() {
 			}
 		}
 		// re-plan the scheduled cut to the moment when the next entry should be cut down
-		return oldestNotCutEntryTouchTime > -1L ? (now - oldestNotCutEntryTouchTime) + 1 : -1L;
+		return oldestNotCutEntryTouchTime > -1L ? (oldestNotCutEntryTouchTime - threshold) + 1 : -1L;
 	}
 
 	@RequiredArgsConstructor

evita_store/evita_store_server/src/main/java/io/evitadb/store/wal/CatalogWriteAheadLog.java

@@ -1132,14 +1132,14 @@ private Stream<Mutation> getCommittedMutationStream(long catalogVersion, boolean
 	 * also re-plans the next cache cut if the cache is not empty.
 	 */
 	private long cutWalCache() {
-		final long now = System.currentTimeMillis();
+		final long threshold = System.currentTimeMillis() - CUT_WAL_CACHE_AFTER_INACTIVITY_MS;
 		long oldestNotCutEntryTouchTime = -1L;
 		for (TransactionLocations locations : transactionLocationsCache.values()) {
 			// if the entry was not cut already (contains more than single / last read record)
 			final int size = locations.size();
 			final boolean oldestRecord = oldestNotCutEntryTouchTime == -1L || locations.getLastReadTime() < oldestNotCutEntryTouchTime;
 			if (size > 1) {
-				if (locations.getLastReadTime() - CUT_WAL_CACHE_AFTER_INACTIVITY_MS > now) {
+				if (locations.getLastReadTime() < threshold) {
 					if (!locations.cut() && oldestRecord) {
 						oldestNotCutEntryTouchTime = locations.getLastReadTime();
 					}
@@ -1152,7 +1152,7 @@ private long cutWalCache() {
 			}
 		}
 		// re-plan the scheduled cut to the moment when the next entry should be cut down
-		return oldestNotCutEntryTouchTime > -1L ? (now - oldestNotCutEntryTouchTime) + 1 : -1L;
+		return oldestNotCutEntryTouchTime > -1L ? (oldestNotCutEntryTouchTime - threshold) + 1 : -1L;
 	}
 
 	/**

evita_test_support/src/main/resources/evita-configuration.yaml

@@ -18,6 +18,8 @@ storage:
   outputBufferSize: ${storage.outputBufferSize:4MB}
   maxOpenedReadHandles: ${storage.maxOpenedReadHandles:12}
   computeCRC32C: ${storage.computeCRC32C:true}
+  minimalActiveRecordShare: ${storage.minimalActiveRecordShare:0.5}
+  fileSizeCompactionThresholdBytes: ${storage.fileSizeCompactionThresholdBytes:100M}
 
 transaction:
   transactionWorkDirectory: ${transaction.workDirectory:null}
@@ -81,4 +83,16 @@ api:
       tlsEnabled: ${api.endpoints.lab.tlsEnabled:true}
       allowedOrigins: ${api.endpoints.lab.allowedOrigins:null}
       gui:
-        enabled: false
+        enabled: false
+        readOnly: ${api.endpoints.lab.gui.readOnly:false}
+        preconfiguredConnections: !include ${api.endpoints.lab.gui.preconfiguredConnections:null}
+    observability:
+      enabled: false
+      host: ${api.endpoints.observability.host:localhost:5557}
+      exposedHost: ${api.endpoints.observability.exposedHost:null}
+      tlsEnabled: ${api.endpoints.observability.tlsEnabled:false}
+      allowedOrigins: ${api.endpoints.observability.allowedOrigins:null}
+      tracing:
+        endpoint: ${api.endpoints.observability.tracing.endpoint:null}
+        protocol: ${api.endpoints.observability.tracing.protocol:grpc}
+      allowedEvents: !include ${api.endpoints.observability.allowedEvents:null}