Fix zombie process race condition in sidecar container (#1806)

* Fix zombie process race condition in sidecar container

Replace custom reapZombies implementation with tini as PID 1 to eliminate
race condition causing "waitid: no child processes" errors during template
operations.

Problem:
- reapZombies() used syscall.Wait4(-1, ...) which reaps ANY child process
- This interfered with normal parent-child process waiting in CmdExec.Run
- Race condition: reapZombies could reap ytt/vendir/imgpkg processes before
  their actual parent (sidecar process) could wait for them
- Result: cmd.Wait() failed with ECHILD ("waitid: no child processes")

Solution:
- Install and use tini as proper PID 1 init system in Dockerfile
- Remove problematic reapZombies function entirely
- tini correctly handles only orphaned processes, not normal children
- Eliminates race condition while maintaining proper zombie cleanup

Changes:
- Dockerfile: Install tini package and set as entrypoint
- sidecarexec.go: Remove reapZombies function and unused imports
- deployment.yml: Add documentation comment about tini configuration

This fixes intermittent failures during PackageRepository reconciliation
and other template-heavy operations under concurrent load.

Fixes: Race condition between zombie reaper and command execution
Made-with: Cursor
Signed-off-by: Marin Dzhigarov <m.dzhigarov@gmail.com>
Made-with: Cursor

* Fix ytt template comment syntax in deployment.yml

Use ytt-specific comment syntax (#!) instead of regular comments (#)
to avoid template compilation errors.

Signed-off-by: Marin Dzhigarov <m.dzhigarov@gmail.com>
Made-with: Cursor

---------

Signed-off-by: Marin Dzhigarov <m.dzhigarov@gmail.com>

Author: mdzhigarov

Dockerfile

@@ -20,8 +20,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
 # --- run image ---
 FROM photon:5.0
 
-# Install openssh for git
-RUN tdnf install -y git openssh-clients
+# Install openssh for git and tini for proper PID 1 handling
+RUN tdnf install -y git openssh-clients tini
 
 # Create the kapp-controller user in the root group, the home directory will be mounted as a volume
 RUN echo "kapp-controller:x:1000:0:/home/kapp-controller:/usr/sbin/nologin" > /etc/passwd
@@ -36,4 +36,5 @@ COPY --from=deps /workspace/out/* ./
 # Run as kapp-controller by default, will be overridden to a random uid on OpenShift
 USER 1000
 ENV PATH="/:${PATH}"
-ENTRYPOINT ["/kapp-controller"]
+# Use tini as PID 1 to properly handle zombie processes and signals
+ENTRYPOINT ["tini", "--", "/kapp-controller"]

cmd/controller/sidecarexec.go

@@ -4,20 +4,16 @@
 package main
 
 import (
-	"syscall"
-	"time"
-
 	"carvel.dev/kapp-controller/pkg/exec"
 	"carvel.dev/kapp-controller/pkg/sidecarexec"
-	"github.com/go-logr/logr"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 func sidecarexecMain() {
 	mainLog := zap.New(zap.UseDevMode(false)).WithName("kc-sidecarexec")
 	mainLog.Info("start sidecarexec", "version", Version)
 
-	go reapZombies(mainLog)
+	// Note: Zombie reaping is now handled by tini as PID 1
 
 	localCmdRunner := exec.NewPlainCmdRunner()
 	opts := sidecarexec.ServerOpts{
@@ -36,18 +32,3 @@ func sidecarexecMain() {
 		mainLog.Error(err, "Serving RPC")
 	}
 }
-
-func reapZombies(log logr.Logger) {
-	log.Info("starting zombie reaper")
-
-	for {
-		var status syscall.WaitStatus
-
-		pid, _ := syscall.Wait4(-1, &status, syscall.WNOHANG, nil)
-		if pid <= 0 {
-			time.Sleep(1 * time.Second)
-		} else {
-			continue
-		}
-	}
-}

config/config/deployment.yml

@@ -67,6 +67,7 @@ spec:
       - name: kapp-controller-sidecarexec
         image: kapp-controller
         args: ["--sidecarexec"]
+        #! tini is already configured as ENTRYPOINT in Dockerfile
         resources:
           requests:
             cpu: 120m