add auto restart

Taskfile.yml

@@ -77,6 +77,7 @@ tasks:
     - cargo clippy --package compiler-core --features wasm --all-targets -- {{.CLIPPY_FLAGS}}
     - cargo clippy --package compiler-wasm --all-targets -- {{.CLIPPY_FLAGS}}
     - cargo clippy --package celery --all-targets -- {{.CLIPPY_FLAGS}}
+    - cargo clippy --package celery-boot --all-targets -- {{.CLIPPY_FLAGS}}
     - cargo fmt --check
 
   fix:rs:

docker/Dockerfile

@@ -3,11 +3,11 @@ EXPOSE 80
 ENV APP_DIR=/opt/app
 COPY ./dist $APP_DIR
 RUN chmod +x $APP_DIR/bin/celery
+RUN chmod +x $APP_DIR/bin/celery-boot
 
 WORKDIR $APP_DIR
 
 ENV CELERSERVER_LOG=INFO \
-    CELERSERVER_ANSI=false \
     CELERSERVER_PORT=80 \
     CELERSERVER_DOCS_DIR=/opt/app/docs \
     CELERSERVER_APP_DIR=/opt/app/app \

server/Taskfile.yml

@@ -1,20 +1,13 @@
 version: '3'
 
 tasks:
-  dev:
-    desc: Start server in watch mode
-    aliases: [d]
-    dotenv:
-    - dev.env
-    cmds:
-    - cargo watch -B 1 -s "cargo run --bin celery {{.CLI_ARGS}}"
-
   run:
     desc: Start server
     dotenv:
     - dev.env
     cmds:
-    - cargo run --bin celery {{.CLI_ARGS}}
+    - cargo build --bin celery {{.CLI_ARGS}}
+    - cargo run --manifest-path ./boot/Cargo.toml --bin celery-boot {{.CLI_ARGS}}
 
   watch:
     desc: Run server tests in watch mode

server/boot/src/main.rs

@@ -2,30 +2,37 @@ use std::path::Path;
 use std::process::{Command, ExitStatus, Stdio};
 
 const VERSION_FILE: &str = "./VERSION";
-const CELERY_BIN: &str = "./celery";
 
 fn main() {
     let version = std::fs::read_to_string(VERSION_FILE)
         .map(|x| x.trim().to_string())
         .unwrap_or_else(|_| "0.0.0-dev unknown".to_string());
-
-    if !Path::new(CELERY_BIN).exists() {
-        println!("Server executable not found!");
+    println!("[boot] server version is {version}");
+
+    let server_executable = std::env::args()
+        .next()
+        .as_ref()
+        .map(Path::new)
+        .and_then(|x| x.parent())
+        .map(|x| x.join("celery"))
+        .expect("fail to get server executable path");
+
+    if !server_executable.exists() {
+        println!("[boot] server executable not found!");
         return;
     }
 
     let mut status: ExitStatus;
-    println!("Starting server...");
+    println!("[boot] starting server...");
 
     loop {
-        status = Command::new(CELERY_BIN)
+        status = Command::new(&server_executable)
             .env("CELERSERVER_VERSION", version.clone())
-            .env("CELERSERVER_BOOTSTRAP", "true")
             .stdin(Stdio::inherit())
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .status()
-            .expect("Fail to spawn server process");
+            .expect("fail to spawn server process");
 
         if !status.success() {
             break;

server/dev.env

@@ -1,5 +1,4 @@
 CELERSERVER_LOG=INFO
-CELERSERVER_ANSI=true
 CELERSERVER_PORT=8173
 CELERSERVER_DOCS_DIR=../docs/src/.vitepress/dist
 CELERSERVER_APP_DIR=../web-client/dist

server/src/env.rs

@@ -10,10 +10,6 @@ pub struct Environment {
     #[envconfig(from = "CELERSERVER_VERSION", default = "0.0.0-dev unknown")]
     pub version: String,
 
-    /// If server is booted with the bootstrap launcher
-    #[envconfig(from = "CELERSERVER_BOOTSTRAP", default = "false")]
-    pub bootstrap: bool,
-
     /// Logging level
     #[envconfig(from = "CELERSERVER_LOG", default = "INFO")]
     pub logging_level: Level,
@@ -22,10 +18,6 @@ pub struct Environment {
     /// Port to listen on
     pub port: u16,
 
-    /// If ANSI formatting is enabled in logs
-    #[envconfig(from = "CELERSERVER_ANSI", default = "true")]
-    pub ansi: bool,
-
     #[envconfig(from = "CELERSERVER_DOCS_DIR")]
     /// Directory to serve docs
     pub docs_dir: String,

server/src/main.rs

@@ -4,13 +4,20 @@
 //! It is recommended to use a reverse proxy such as nginx to handle HTTPS.
 //! Alternatively, you can use a CDN such as Cloudflare to proxy the website.
 
-use axum::response::Redirect;
-use axum::{routing, Router};
-use axum_server::Server;
-use std::io;
+use std::io::{self, IsTerminal};
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use axum::extract::{Request, State};
+use axum::middleware::Next;
+use axum::response::{Redirect, Response};
+use axum::{middleware, routing, Router};
+use axum_server::{Handle, Server};
+use tokio::sync::Mutex;
+use tokio::time::{self, Duration, Instant};
+use tower::ServiceBuilder;
 use tower_http::services::{ServeDir, ServeFile};
-use tower_http::trace::{DefaultMakeSpan, DefaultOnRequest, DefaultOnResponse};
+use tower_http::trace::{DefaultMakeSpan, DefaultOnRequest, DefaultOnResponse, TraceLayer};
 use tracing::{debug, info, Level};
 
 mod api;
@@ -26,7 +33,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let env = Environment::parse();
     tracing_subscriber::fmt()
         .compact()
-        .with_ansi(env.ansi)
+        .with_ansi(std::io::stdout().is_terminal())
         .with_max_level(env.logging_level)
         .init();
     info!("celer server version: {}", env.version);
@@ -46,6 +53,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
     info!("configuring routes...");
 
+    let globals = Globals::new();
+
     let router = Router::new();
     let router = init_home(router);
     let router = init_docs(router, &env.docs_dir)?;
@@ -57,26 +66,37 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     )?;
     let router = api::init_api(router, &env)?;
 
+    let shutdown = middleware::from_fn_with_state(globals.clone(), shutdown_middleware);
+
     let router = router.layer(
-        tower_http::trace::TraceLayer::new_for_http()
-            .make_span_with(DefaultMakeSpan::new().level(Level::INFO))
-            .on_request(DefaultOnRequest::new().level(Level::INFO))
-            .on_response(DefaultOnResponse::new().level(Level::INFO)),
+        ServiceBuilder::new()
+            .layer(
+                TraceLayer::new_for_http()
+                    .make_span_with(DefaultMakeSpan::new().level(Level::INFO))
+                    .on_request(DefaultOnRequest::new().level(Level::INFO))
+                    .on_response(DefaultOnResponse::new().level(Level::INFO)),
+            )
+            .layer(shutdown),
     );
 
     let address = format!("0.0.0.0:{}", env.port).parse()?;
+    let handle = globals.handle;
     if let Some(tls_config) = env.get_https_config().await {
         info!("starting server on https://{address}");
         axum_server::bind_rustls(address, tls_config)
+            .handle(handle)
             .serve(router.into_make_service())
             .await?;
     } else {
         info!("starting server on http://{address}");
         Server::bind(address)
+            .handle(handle)
             .serve(router.into_make_service())
             .await?;
     }
 
+    info!("server stopped");
+
     Ok(())
 }
 
@@ -132,3 +152,49 @@ fn init_static(
     }
     Ok(router)
 }
+
+/// Global state for server
+#[derive(Clone)]
+struct Globals {
+    pub handle: Handle,
+    pub last_shutdown_check: Arc<Mutex<Instant>>,
+}
+
+impl Globals {
+    pub fn new() -> Self {
+        Self {
+            handle: Handle::new(),
+            last_shutdown_check: Arc::new(Mutex::new(Instant::now())),
+        }
+    }
+}
+
+async fn shutdown_middleware(State(globals): State<Globals>, req: Request, next: Next) -> Response {
+    let restart_interval = Duration::from_secs(15 * 60); // 15 minutes
+    let response = next.run(req).await;
+    let mut last_shutdown_check = globals.last_shutdown_check.lock().await;
+    let now = Instant::now();
+    let should_restart = now.duration_since(*last_shutdown_check) > restart_interval;
+    *last_shutdown_check = now;
+    if should_restart {
+        info!("server has idled for too long, queueing restart...");
+        let last_shutdown_check = globals.last_shutdown_check.clone();
+        let handle = globals.handle.clone();
+        tokio::spawn(async move {
+            loop {
+                time::sleep(Duration::from_secs(5)).await;
+                {
+                    let last_shutdown_check = last_shutdown_check.lock().await;
+                    let now = Instant::now();
+                    if now.duration_since(*last_shutdown_check) > Duration::from_secs(5) {
+                        break;
+                    }
+                }
+            }
+            info!("gracefully shutting down server...");
+            handle.graceful_shutdown(Some(Duration::from_secs(15)));
+        });
+    }
+
+    response
+}