feat: add connection draining during shutdown

bin/varnishd/cache/cache_cli.c

@@ -39,6 +39,7 @@
 #include "config.h"
 
 #include "cache_varnishd.h"
+#include "acceptor/cache_acceptor.h"
 #include "common/heritage.h"
 
 #include "vcli_serve.h"
@@ -108,16 +109,43 @@ CLI_Run(void)
 	cli->auth = 255;	// Non-zero to disable paranoia in vcli_serve
 
 	do {
-		i = VCLS_Poll(cache_cls, cli, -1);
+		/*
+		 * When draining, use short poll timeout so we can check
+		 * if all connections have been closed.
+		 */
+		i = VCLS_Poll(cache_cls, cli, cache_draining ? 1000 : -1);
+		if (cache_draining && SES_ActiveCount() == 0 && VCL_Drained()) {
+			VSL(SLT_CLI, NO_VXID,
+			    "All connections drained, shutting down");
+			break;
+		}
 	} while (i == 0);
 	VSL(SLT_CLI, NO_VXID, "EOF on CLI connection, worker stops");
 }
 
+static void v_matchproto_(cli_func_t)
+ccf_drain(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)cli;
+	(void)av;
+	(void)priv;
+	cache_draining = 1;
+	/* Enable rapid VCL release so idle workers release VCL immediately */
+	cache_param->debug_bits[DBG_VCLREL >> 3] |=
+	    (0x80 >> (DBG_VCLREL & 7));
+	/* Stop accepting new connections immediately */
+	VCA_Shutdown();
+	/* Wake up idle workers so they release VCL references */
+	Pool_WakeIdle();
+	VCLI_Out(cli, "Connection draining enabled, new connections refused");
+}
+
 /*--------------------------------------------------------------------*/
 
 static struct cli_proto cli_cmds[] = {
 	{ CLICMD_PING,	"i", VCLS_func_ping, VCLS_func_ping_json },
 	{ CLICMD_HELP,	"i", VCLS_func_help, VCLS_func_help_json },
+	{ CLICMD_SERVER_DRAIN,	"", ccf_drain },
 	{ NULL }
 };
 

bin/varnishd/cache/cache_main.c

@@ -54,6 +54,7 @@
 #include "hash/hash_slinger.h"
 
 volatile struct params		*cache_param;
+volatile unsigned		cache_draining = 0;
 static pthread_mutex_t		cache_vrnd_mtx;
 static vtim_dur			shutdown_delay = 0;
 
@@ -557,9 +558,13 @@ child_main(int sigmagic, size_t altstksz)
 	if (shutdown_delay > 0)
 		VTIM_sleep(shutdown_delay);
 
-	VCA_Shutdown();
+	/* If draining, VCA_Shutdown was already called in ccf_drain */
+	if (!cache_draining)
+		VCA_Shutdown();
 	BAN_Shutdown();
 	STV_warn();
+	/* Wake up any idle workers so they release VCL references */
+	Pool_WakeIdle();
 	VCL_Shutdown();
 	EXP_Shutdown();
 	STV_close();

bin/varnishd/cache/cache_pool.c

@@ -75,6 +75,32 @@ Pool_TrySumstat(const struct worker *wrk)
 	return (1);
 }
 
+/*--------------------------------------------------------------------
+ * Wake up all idle workers so they can release VCL references quickly.
+ * Called when draining starts.
+ */
+
+void
+Pool_WakeIdle(void)
+{
+	struct pool *pp;
+	struct pool_task *pt;
+	struct worker *wrk;
+
+	Lck_Lock(&pool_mtx);
+	VTAILQ_FOREACH(pp, &pools, list) {
+		CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+		Lck_Lock(&pp->mtx);
+		VTAILQ_FOREACH(pt, &pp->idle_queue, list) {
+			AZ(pt->func);
+			CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
+			PTOK(pthread_cond_signal(&wrk->cond));
+		}
+		Lck_Unlock(&pp->mtx);
+	}
+	Lck_Unlock(&pool_mtx);
+}
+
 /*--------------------------------------------------------------------
  * Facility for scheduling a task on any convenient pool.
  */

bin/varnishd/cache/cache_session.c

@@ -52,6 +52,9 @@
 #include "vtim.h"
 #include "waiter/waiter.h"
 
+/* Count of active sessions (for drain detection) */
+static volatile unsigned n_sess_active = 0;
+
 static const struct {
 	const char		*type;
 } sess_attr[SA_LAST] = {
@@ -447,6 +450,7 @@ SES_New(struct pool *pp)
 	sp->idle_send_timeout = NAN;
 	Lck_New(&sp->mtx, lck_sess);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	n_sess_active++;
 	return (sp);
 }
 
@@ -689,6 +693,8 @@ SES_Rel(struct sess *sp)
 #ifdef ENABLE_WORKSPACE_EMULATOR
 	WS_Rollback(sp->ws, 0);
 #endif
+	assert(n_sess_active > 0);
+	n_sess_active--;
 	MPL_Free(sp->pool->mpl_sess, sp);
 }
 
@@ -720,3 +726,12 @@ SES_DestroyPool(struct pool *pp)
 	MPL_Destroy(&pp->mpl_sess);
 	Waiter_Destroy(&pp->waiter);
 }
+
+/*
+ * Return count of active sessions (for drain detection).
+ */
+unsigned
+SES_ActiveCount(void)
+{
+	return (n_sess_active);
+}

bin/varnishd/cache/cache_varnishd.h

@@ -122,6 +122,7 @@ typedef enum htc_status_e htc_complete_f(struct http_conn *);
 /* -------------------------------------------------------------------*/
 
 extern volatile struct params * cache_param;
+extern volatile unsigned cache_draining;
 
 /* -------------------------------------------------------------------
  * The VCF facility is deliberately undocumented, use at your peril.
@@ -414,6 +415,7 @@ int Pool_Task_Arg(struct worker *, enum task_prio, task_func_t *,
     const void *arg, size_t arg_len);
 void Pool_Sumstat(const struct worker *w);
 int Pool_TrySumstat(const struct worker *wrk);
+void Pool_WakeIdle(void);
 void Pool_PurgeStat(unsigned nobj);
 int Pool_Task_Any(struct pool_task *task, enum task_prio prio);
 void pan_pool(struct vsb *);
@@ -456,6 +458,7 @@ void SES_DestroyPool(struct pool *);
 void SES_Wait(struct sess *, const struct transport *);
 void SES_Ref(struct sess *sp);
 void SES_Rel(struct sess *sp);
+unsigned SES_ActiveCount(void);
 
 void HTC_Status(enum htc_status_e, const char **, const char **);
 void HTC_RxInit(struct http_conn *htc, struct ws *ws);
@@ -504,6 +507,7 @@ struct vsb *VCL_Rel_CliCtx(struct vrt_ctx **);
 void VCL_Panic(struct vsb *, const char *nm, const struct vcl *);
 void VCL_Poll(void);
 void VCL_Init(void);
+int VCL_Drained(void);
 void VCL_Shutdown(void);
 
 #define VCL_MET_MAC(l,u,t,b) \

bin/varnishd/cache/cache_vcl.c

@@ -1288,6 +1288,27 @@ VCL_Init(void)
 	VSL_Setup(&vsl_cli, NULL, 0);
 }
 
+/*
+ * Check if all VCL references have been released.
+ * Returns 1 if all VCLs have busy == 0, 0 otherwise.
+ */
+int
+VCL_Drained(void)
+{
+	struct vcl *vcl;
+	int drained = 1;
+
+	Lck_Lock(&vcl_mtx);
+	VTAILQ_FOREACH(vcl, &vcl_head, list) {
+		if (vcl->busy) {
+			drained = 0;
+			break;
+		}
+	}
+	Lck_Unlock(&vcl_mtx);
+	return (drained);
+}
+
 void
 VCL_Shutdown(void)
 {

bin/varnishd/cache/cache_wrk.c

@@ -446,6 +446,10 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 					tmo = INFINITY;
 				else if (DO_DEBUG(DBG_VTC_MODE))
 					tmo = now + 1.;
+				else if (cache_draining)
+					/* During draining, use short timeout
+					 * to release VCL quickly */
+					tmo = now + 1.;
 				else
 					tmo = now + 60.;
 				(void)Lck_CondWaitUntil(

bin/varnishd/http1/cache_http1_deliver.c

@@ -92,6 +92,13 @@ V1D_Deliver(struct req *req, int sendbody)
 	} else if (!http_GetHdr(req->resp, H_Connection, NULL))
 		http_SetHeader(req->resp, "Connection: keep-alive");
 
+	/* If draining, force Connection: close */
+	if (cache_draining && req->doclose == SC_NULL) {
+		req->doclose = SC_DRAIN;
+		http_Unset(req->resp, H_Connection);
+		http_SetHeader(req->resp, "Connection: close");
+	}
+
 	CHECK_OBJ_NOTNULL(req->wrk, WORKER_MAGIC);
 
 	v1l = V1L_Open(req->wrk->aws, &req->sp->fd, req->vsl,

bin/varnishd/mgt/mgt.h

@@ -64,6 +64,7 @@ struct vclprog;
 extern struct vev_root	*mgt_evb;
 extern unsigned		d_flag;
 extern int		exit_status;
+extern int		mgt_draining;
 
 /* option=value argument parse helper */
 static inline const char *

bin/varnishd/mgt/mgt_child.c

@@ -628,11 +628,17 @@ mgt_reap_child(void)
 		exit(1);
 	}
 
-	if (child_state == CH_DIED && mgt_param.auto_restart)
+	if (child_state == CH_DIED && mgt_param.auto_restart && !mgt_draining)
 		mgt_launch_child(NULL);
-	else if (child_state == CH_DIED)
+	else if (child_state == CH_DIED) {
 		child_state = CH_STOPPED;
-	else if (child_state == CH_STOPPING)
+		if (mgt_draining) {
+			/* Child exited during drain, proceed with shutdown */
+			MGT_Complain(C_INFO,
+			    "Child exited during drain, proceeding with shutdown");
+			(void)raise(SIGTERM);
+		}
+	} else if (child_state == CH_STOPPING)
 		child_state = CH_STOPPED;
 }
 

bin/varnishd/mgt/mgt_main.c

@@ -74,6 +74,8 @@ struct vsb		*vident;
 struct VSC_mgt		*VSC_C_mgt;
 static int		I_fd = -1;
 static char		*workdir;
+int			mgt_draining = 0;
+static struct vev	*drain_timer = NULL;
 
 static struct vfil_path *vcl_path = NULL;
 
@@ -444,17 +446,66 @@ mgt_eric_im_done(int eric_fd, unsigned u)
 
 /*--------------------------------------------------------------------*/
 
+static int v_matchproto_(vev_cb_f)
+mgt_drain_timeout(const struct vev *e, int what)
+{
+	(void)e;
+	(void)what;
+	MGT_Complain(C_INFO, "Drain timeout reached, stopping child");
+	MCH_Stop_Child();
+	(void)raise(SIGTERM);  /* Trigger signal handler to exit event loop */
+	return (1);  /* Remove this timeout event */
+}
+
 static int v_matchproto_(vev_cb_f)
 mgt_sigint(const struct vev *e, int what)
 {
+	unsigned status;
+	char *p;
 
 	(void)what;
 	MGT_Complain(C_INFO, "Manager got %s from PID %jd",
 	    e->name, (intmax_t)e->siginfo->si_pid);
 	(void)fflush(stdout);
-	if (MCH_Running())
+
+	if (!MCH_Running())
+		return (-42);
+
+	/* If drain_timeout is 0, do immediate shutdown */
+	if (mgt_param.drain_timeout == 0) {
 		MCH_Stop_Child();
-	return (-42);
+		return (-42);
+	}
+
+	/* If already draining, proceed to shutdown */
+	if (mgt_draining) {
+		MGT_Complain(C_INFO, "Already draining, stopping child");
+		MCH_Stop_Child();
+		return (-42);
+	}
+
+	/* Send drain command to child */
+	if (mgt_cli_askchild(&status, &p, "drain\n")) {
+		MGT_Complain(C_ERR, "Failed to send drain command: %s", p);
+		free(p);
+		MCH_Stop_Child();
+		return (-42);
+	}
+	free(p);
+
+	mgt_draining = 1;
+	MGT_Complain(C_INFO, "Connection draining started, timeout %.0f seconds",
+	    mgt_param.drain_timeout);
+
+	/* Set up drain timeout event */
+	drain_timer = VEV_Alloc();
+	AN(drain_timer);
+	drain_timer->name = "drain_timeout";
+	drain_timer->timeout = mgt_param.drain_timeout;
+	drain_timer->callback = mgt_drain_timeout;
+	AZ(VEV_Start(mgt_evb, drain_timer));
+
+	return (0);  /* Keep event loop running */
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishtest/tests/b00099.vtc

@@ -0,0 +1,46 @@
+varnishtest "Test connection draining adds Connection: close"
+
+# Test that during draining, responses include Connection: close header
+# The connection must be established BEFORE drain starts (VCA_Shutdown stops new connections)
+
+barrier b1 cond 2
+barrier b2 cond 2
+
+server s1 {
+	rxreq
+	# Signal that backend received the request
+	barrier b1 sync
+	# Wait for drain to start before responding
+	barrier b2 sync
+	txresp -body "response during drain"
+} -start
+
+varnish v1 -arg "-p drain_timeout=10" -vcl+backend { } -start
+
+# Start request (establishes connection)
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "response during drain"
+	# Response during drain should have Connection: close
+	expect resp.http.Connection == "close"
+} -start
+
+# Wait for request to reach backend (connection is now established)
+barrier b1 sync
+
+# Now start draining - connection already exists
+shell "kill -15 ${v1_pid}"
+
+# Small delay for drain to initialize
+delay 0.1
+
+# Signal backend to respond
+barrier b2 sync
+
+# Wait for client to complete
+client c1 -wait
+
+# Use -cleanup since varnish will exit on its own after drain completes
+varnish v1 -cleanup