feat: make lake run with non-root users (apache#5583)

* feat: make devlake run in nonroot user

* feat: make devlake run as nonroot user and specify the user id

* feat: make lake run with non-root user devlake whose home dir is /app

* fix: adjust path

* feat: support for debug builds and plugin-whitelisting in Dockerfile for easier troubleshooting

---------

Co-authored-by: Keon Amini <keon.amini@merico.dev>

Makefile

@@ -71,9 +71,6 @@ swag:
 build-plugin:
 	make build-plugin -C backend
 
-build-plugin-debug:
-	make build-plugin-debug -C backend
-
 build-worker:
 	make build-worker -C backend
 

backend/Dockerfile

@@ -60,6 +60,7 @@ COPY --from=debian-arm64 /usr/include /rootfs-arm64/usr/include
 COPY --from=debian-arm64 /usr/lib/aarch64-linux-gnu /rootfs-arm64/usr/lib/aarch64-linux-gnu
 COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-gnu
 
+
 RUN for arch in aarch64 x86_64 ; do \
         mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \
         wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.2.tar.gz -O - | tar -xz && \
@@ -87,6 +88,8 @@ ENV GOBIN=/app/bin
 ARG TARGETPLATFORM
 ARG TAG=
 ARG SHA=
+ARG GO_PLUGINS=
+ARG DEBUG=
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
     if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
@@ -100,7 +103,11 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     fi && \
     export PKG_CONFIG_PATH=/usr/local/deps/target/lib/pkgconfig && \
     export CGO_ENABLED=1 &&\
-    make all
+    PLUGIN="$GO_PLUGINS" make all
+
+RUN if [ "$DEBUG" = "true" ]; then \
+      go install github.com/go-delve/delve/cmd/dlv@latest; \
+    fi;
 
 # remove symlink in lib, we will recreate in final image
 RUN cd /usr/local/deps/target/lib && \
@@ -113,10 +120,8 @@ RUN cd /usr/local/deps/target/lib && \
 
 FROM python:3.9-slim-bullseye as base
 
-ENV PYTHONUNBUFFERED=1
-
 RUN apt-get update && \
-    apt-get install -y python3-dev python3-pip tar curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev && \
+    apt-get install -y python3-dev python3-pip tar curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini && \
     apt-get clean && \
     rm -fr /usr/share/doc/* \
            /usr/share/info/* \
@@ -132,38 +137,46 @@ RUN apt-get update && \
 
 EXPOSE 8080
 
+RUN useradd -ms /bin/bash -d /app devlake -u 1010
+USER devlake
+ENV PYTHONUNBUFFERED=1
+
 WORKDIR /app
+
+RUN mkdir logs
+VOLUME /app/logs
+
 # Setup Python
 COPY python/ /app/python/
 RUN python3 -m pip install --no-cache --upgrade pip setuptools && \
-    python3 -m pip install --no-cache -r python/requirements.txt && \
-    rm -fr /usr/share/python-wheels/*
+    python3 -m pip install --no-cache -r python/requirements.txt
+
 # Setup Python Poetry package manager
 RUN curl -sSL https://install.python-poetry.org | python3 -
-RUN ln -sf /root/.local/bin/poetry /usr/local/bin
-# Build Python plugins
-RUN find /app/python/ -name "*.sh" | xargs -I{} chmod +x {}
-RUN /app/python/build.sh
+ENV PATH="$PATH:/app/.local/bin"
 
+# Build Python plugins, make sure the scripts has execute permission
+# RUN find /app/python/ -name "*.sh" | xargs -I{} chmod +x {}
+RUN /app/python/build.sh
 
 FROM base as devlake-base
+ARG DEBUG=
 
 # libraries
 ENV LD_LIBRARY_PATH=/app/libs
 RUN mkdir -p /app/libs
 COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs
+COPY --from=build /go/bin /usr/bin
 RUN ldconfig -vn /app/libs
 
 # apps
 COPY --from=build /app/bin /app/bin
 COPY --from=build /app/resources /app/resources
 
 ENV PATH="/app/bin:${PATH}"
+ENV DEBUG="$DEBUG"
 
 #add tini, prevent zombie process
-RUN apt-get update && \
-    apt-get install -y tini
-
 ENTRYPOINT ["/usr/bin/tini", "--"]
 
 CMD ["lake"]

backend/Makefile

@@ -35,20 +35,41 @@ python-dep:
 dep: go-dep python-dep
 
 swag:
-	swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go -g ./plugins/*/api/*.go
-	@echo "visit the swagger document on http://localhost:8080/swagger/index.html"
+	if [ -z $(PLUGIN) ]; then \
+		swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go -g ./plugins/*/api/*.go; \
+	elif [ $(PLUGIN) = "none" ]; then \
+		swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go;\
+	else \
+	  	plugins="";\
+		for p in $$(echo $(PLUGIN) | tr "," "\n"); do \
+			plugins="$$plugins -g ./plugins/$$p/api/*.go"; \
+		done;\
+		swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go "$$plugins"; \
+	fi;\
+	echo "visit the swagger document on http://localhost:8080/swagger/index.html";
 
 build-plugin:
-	@sh scripts/compile-plugins.sh
-
-build-plugin-debug:
-	@sh scripts/compile-plugins.sh -gcflags='all=-N -l'
+	if [ "$(PLUGIN)" = "none" ]; then \
+  		echo "Building plugins will be skipped"; \
+	elif [ "$(DEBUG)" = "true" ]; then \
+  		PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh -gcflags='all=-N -l'; \
+  	else \
+		PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh; \
+	fi
 
 build-worker:
-	go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/
+	if [ "$(DEBUG)" = "true" ]; then \
+  		go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/; \
+  	else \
+		go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/; \
+	fi
 
 build-server: swag
-	go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/
+	if [ "$(DEBUG)" = "true" ]; then \
+  		go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
+  	else \
+		go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
+	fi
 
 build-python: #don't mix this with the other build commands
 	find ./python/ -name "*.sh" | xargs chmod +x &&\

backend/impls/logruslog/stream.go

@@ -28,7 +28,7 @@ func GetFileStream(path string) (io.Writer, errors.Error) {
 	if path == "" {
 		return os.Stdout, nil
 	}
-	err := os.MkdirAll(filepath.Dir(path), os.ModePerm)
+	err := os.MkdirAll(filepath.Dir(path), 0777)
 	if err != nil {
 		return nil, errors.Convert(err)
 	}

backend/scripts/compile-plugins.sh

@@ -38,9 +38,12 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 PLUGIN_SRC_DIR=$SCRIPT_DIR/../plugins
 PLUGIN_OUTPUT_DIR=$SCRIPT_DIR/../bin/plugins
 
+
 if [ -z "$PLUGIN" ]; then
+    echo "Building all plugins"
     PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -empty)
 else
+    echo "Building the following plugins: $PLUGIN"
     PLUGINS=
     for p in $(echo "$PLUGIN" | tr "," "\n"); do
         PLUGINS="$PLUGINS $PLUGIN_SRC_DIR/$p"
@@ -52,7 +55,7 @@ rm -rf $PLUGIN_OUTPUT_DIR/*
 PIDS=""
 for PLUG in $PLUGINS; do
     NAME=$(basename $PLUG)
-    echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so"
+    echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so with args: $*"
     go build -buildmode=plugin "$@" -o $PLUGIN_OUTPUT_DIR/$NAME/$NAME.so $PLUG/*.go &
     PIDS="$PIDS $!"
     # avoid too many processes causing signal killed