[Fix] --json-report: field parity, unified JSON escaping, O(N) list rendering at scale; issue #482

rfxn · rfxn · commit 72ff465f5cc1 · 2026-04-17T02:19:17.000-05:00
[Fix] --json-report list: reports[] entries now include a "path" field, matching the text-mode list output; active[] entries gain the lifecycle schema (eta, workers, sig_version, progress{}) so --json-report list and -L/--list-active emit the same shape; stopped[] entries gain an elapsed field; all string fields in active[] and stopped[] are now JSON-escaped via the shared _json_escape_string helper (previously only "path" was escaped — stage, engine, stopped_hr, stages, sig_version could emit invalid JSON if scan.meta contained quotes, backslashes, or control characters); issue #482 [Fix] --json-report list: scaling regression at large session counts. Pre-fix the function exhibited effective hang at ~20K indexed sessions from two compounding costs: (1) _seen_ids built as a whitespace-joined string with glob-pattern dedup (O(N^2)) and (2) one subshell fork per report for path escaping. Dedup now uses a function-scoped local -A associative array (O(N)), and the reports[]/legacy hot loops use the new _json_escape_var out-parameter helper instead of $() command substitution. Measured: 20K reports 82s -> 1.7s; 50K reports ~1.7s (linear). Regression guard: tests/31-json-report.bats test 27 runs a 10K-entry synthetic index under a 30s timeout [Change] Lifecycle JSON (-L --format json): "scan_id" is now the canonical field name; "scanid" remains as a deprecated alias for one release cycle and will be removed in v2.1.0. Consumers should switch to "scan_id" [Change] Lifecycle JSON (-L --format json): "workers" field type normalized to unquoted number to match the field's underlying integer value [Change] _json_escape_string: promoted from lmd_hook.sh (optional sub-lib) to lmd.lib.sh shared utilities so all JSON emitters — scan list, active lifecycle, post-scan hook, and ELK dispatch — share one helper. Three call sites in lmd_alert.sh previously using _alert_json_escape (from the vendored alert_lib) now use the project-owned helper; vendored library coupling reduced to the lib's internal self-use. Sibling out-parameter helper _json_escape_var (sets _JSON_ESC_OUT) also defined for hot loops that must avoid a subshell fork per iteration [New] tests/31-json-report.bats: four regression cases — reports[] path parity (issue #482), JSON validity with special-character paths, total_quarantined field presence, linear scaling at 10K sessions [Change] tests/46-post-scan-hook.bats: J-15 structural guard now points to lmd.lib.sh (the new home for _json_escape_string)
diff --git a/CHANGELOG b/CHANGELOG
@@ -152,6 +152,38 @@ v2.0.1 | Mar 25 2026:
 
   -- Bug Fixes --
 
+[Fix] --json-report list: reports[] entries now include a "path" field, matching
+      the text-mode list output; active[] entries gain the lifecycle schema
+      (eta, workers, sig_version, progress{}) so --json-report list and
+      -L/--list-active emit the same shape; stopped[] entries gain an elapsed
+      field; all string fields in active[] and stopped[] are now JSON-escaped
+      via the shared _json_escape_string helper (previously only "path" was
+      escaped — stage, engine, stopped_hr, stages, sig_version could emit
+      invalid JSON if scan.meta contained quotes, backslashes, or control
+      characters); issue #482
+[Change] Lifecycle JSON (-L --format json): "scan_id" is now the canonical
+      field name; "scanid" remains as a deprecated alias for one release cycle
+      and will be removed in v2.1.0. Consumers should switch to "scan_id"
+[Change] Lifecycle JSON (-L --format json): "workers" field type normalized to
+      unquoted number to match the field's underlying integer value
+[Change] _json_escape_string: promoted from lmd_hook.sh (optional sub-lib) to
+      lmd.lib.sh shared utilities so all JSON emitters — scan list, active
+      lifecycle, post-scan hook, and ELK dispatch — share one helper. Three
+      call sites in lmd_alert.sh previously using _alert_json_escape (from the
+      vendored alert_lib) now use the project-owned helper; vendored library
+      coupling reduced to the lib's internal self-use. A sibling out-parameter
+      helper _json_escape_var (sets _JSON_ESC_OUT) is also defined for hot
+      loops that must avoid a subshell fork per iteration
+[Fix] --json-report list: scaling regression at large session counts. Pre-fix
+      the function exhibited effective hang at ~20K indexed sessions from two
+      compounding costs: (1) _seen_ids built as a whitespace-joined string
+      that grew O(N^2) with glob-pattern dedup, and (2) one subshell fork per
+      report for path escaping. Dedup now uses a function-scoped local -A
+      associative array (O(N) total), and the reports[]/legacy hot loops use
+      the new _json_escape_var out-parameter helper instead of $() command
+      substitution. Measured: 20K reports 82s -> 1.7s; 50K reports now ~1.7s
+      (linear). Regression guard: tests/31-json-report.bats test 27 runs a
+      10K-entry synthetic index under a 30s timeout
 [Fix] clamav_linksigs: guard mktemp staging failure to prevent writing signature
       files into the filesystem root. When mktemp -d fails, the empty _staging
       variable caused "cp -f ... /" as root; pr#478
diff --git a/CHANGELOG.RELEASE b/CHANGELOG.RELEASE
@@ -152,6 +152,38 @@ v2.0.1 | Mar 25 2026:
 
   -- Bug Fixes --
 
+[Fix] --json-report list: reports[] entries now include a "path" field, matching
+      the text-mode list output; active[] entries gain the lifecycle schema
+      (eta, workers, sig_version, progress{}) so --json-report list and
+      -L/--list-active emit the same shape; stopped[] entries gain an elapsed
+      field; all string fields in active[] and stopped[] are now JSON-escaped
+      via the shared _json_escape_string helper (previously only "path" was
+      escaped — stage, engine, stopped_hr, stages, sig_version could emit
+      invalid JSON if scan.meta contained quotes, backslashes, or control
+      characters); issue #482
+[Change] Lifecycle JSON (-L --format json): "scan_id" is now the canonical
+      field name; "scanid" remains as a deprecated alias for one release cycle
+      and will be removed in v2.1.0. Consumers should switch to "scan_id"
+[Change] Lifecycle JSON (-L --format json): "workers" field type normalized to
+      unquoted number to match the field's underlying integer value
+[Change] _json_escape_string: promoted from lmd_hook.sh (optional sub-lib) to
+      lmd.lib.sh shared utilities so all JSON emitters — scan list, active
+      lifecycle, post-scan hook, and ELK dispatch — share one helper. Three
+      call sites in lmd_alert.sh previously using _alert_json_escape (from the
+      vendored alert_lib) now use the project-owned helper; vendored library
+      coupling reduced to the lib's internal self-use. A sibling out-parameter
+      helper _json_escape_var (sets _JSON_ESC_OUT) is also defined for hot
+      loops that must avoid a subshell fork per iteration
+[Fix] --json-report list: scaling regression at large session counts. Pre-fix
+      the function exhibited effective hang at ~20K indexed sessions from two
+      compounding costs: (1) _seen_ids built as a whitespace-joined string
+      that grew O(N^2) with glob-pattern dedup, and (2) one subshell fork per
+      report for path escaping. Dedup now uses a function-scoped local -A
+      associative array (O(N) total), and the reports[]/legacy hot loops use
+      the new _json_escape_var out-parameter helper instead of $() command
+      substitution. Measured: 20K reports 82s -> 1.7s; 50K reports now ~1.7s
+      (linear). Regression guard: tests/31-json-report.bats test 27 runs a
+      10K-entry synthetic index under a 30s timeout
 [Fix] clamav_linksigs: guard mktemp staging failure to prevent writing signature
       files into the filesystem root. When mktemp -d fails, the empty _staging
       variable caused "cp -f ... /" as root; pr#478
diff --git a/files/internals/lmd.lib.sh b/files/internals/lmd.lib.sh
@@ -155,6 +155,25 @@ get_filestat() {
 	md5_hash="${_md5out%% *}"
 }
 
+# Backslash MUST be escaped first — subsequent substitutions insert \ characters
+# that would otherwise get double-escaped.
+#
+# Two call patterns:
+#   $(_json_escape_string "$x")   — readable; forks a subshell (use in cold paths)
+#   _json_escape_var "$x"; use "$_JSON_ESC_OUT"  — zero forks (hot loops)
+_json_escape_var() {
+	_JSON_ESC_OUT="${1//\\/\\\\}"
+	_JSON_ESC_OUT="${_JSON_ESC_OUT//\"/\\\"}"
+	_JSON_ESC_OUT="${_JSON_ESC_OUT//$'\t'/\\t}"
+	_JSON_ESC_OUT="${_JSON_ESC_OUT//$'\r'/\\r}"
+	_JSON_ESC_OUT="${_JSON_ESC_OUT//$'\n'/\\n}"
+}
+
+_json_escape_string() {
+	_json_escape_var "$1"
+	printf '%s' "$_JSON_ESC_OUT"
+}
+
 ## Source vendored libraries
 
 if [ -f "$tlog_lib" ]; then
diff --git a/files/internals/lmd_alert.sh b/files/internals/lmd_alert.sh
@@ -304,9 +304,9 @@ _lmd_elk_post_hits() {
 		[ -z "$_sig" ] && continue
 		[[ "$_sig" == "#"* ]] && continue  # skip header
 		[ "$_quarpath" = "-" ] && _quarpath=""
-		_sig=$(_alert_json_escape "$_sig")
-		_filepath=$(_alert_json_escape "$_filepath")
-		_hash=$(_alert_json_escape "${_hash:--}")
+		_sig=$(_json_escape_string "$_sig")
+		_filepath=$(_json_escape_string "$_filepath")
+		_hash=$(_json_escape_string "${_hash:--}")
 		$curl --output /dev/null --silent --show-error \
 			-XPOST "$elk_url" \
 			-H 'Content-Type: application/json' \
@@ -717,15 +717,28 @@ _lmd_render_json_list() {
 				_lifecycle_read_meta "$_jl_scanid" || continue
 				[ "$_first_active" != "1" ] && printf ","
 				_first_active=0
-				local _jl_path="${_meta_path//\\/\\\\}"
-				_jl_path="${_jl_path//\"/\\\"}"
+				local _jl_path _jl_engine _jl_sig
+				_jl_path=$(_json_escape_string "$_meta_path")
+				_jl_engine=$(_json_escape_string "${_meta_engine:--}")
+				_jl_sig=$(_json_escape_string "${_meta_sig_version:--}")
 				local _jl_pid="${_meta_pid:-0}"; [ "$_jl_pid" = "-" ] && _jl_pid=0
 				local _jl_files="${_meta_total_files:-0}"; [ "$_jl_files" = "-" ] && _jl_files=0
 				local _jl_hits="${_meta_hits:-0}"; [ "$_jl_hits" = "-" ] && _jl_hits=0
 				local _jl_elapsed="${_meta_elapsed:-0}"; [ "$_jl_elapsed" = "-" ] && _jl_elapsed=0
-				printf '\n    {"scan_id": "%s", "state": "%s", "pid": %s, "path": "%s", "engine": "%s", "total_files": %s, "hits": %s, "elapsed": %s}' \
+				# Live elapsed for running scans (matches _lifecycle_render_json_active)
+				if [ "$_jl_elapsed" = "0" ] && [ -n "$_meta_started" ] && [ "$_meta_started" != "0" ]; then
+					_jl_elapsed="$(( $(command date +%s) - _meta_started ))"
+				fi
+				local _jl_workers="${_meta_workers:-0}"; [ "$_jl_workers" = "-" ] && _jl_workers=0
+				local _jl_prog_pos="${_meta_progress_pos:-0}"; [ "$_jl_prog_pos" = "-" ] && _jl_prog_pos=0
+				local _jl_prog_total="${_meta_progress_total:-0}"; [ "$_jl_prog_total" = "-" ] && _jl_prog_total=0
+				local _jl_eta
+				_jl_eta=$(_lifecycle_compute_eta "${_meta_engine:-}" "$_jl_elapsed" "$_jl_prog_pos" "$_jl_prog_total")
+				printf '\n    {"scan_id": "%s", "state": "%s", "pid": %s, "path": "%s", "engine": "%s", "total_files": %s, "hits": %s, "elapsed": %s, "eta": %s, "workers": %s, "sig_version": "%s", "progress": {"position": %s, "total": %s}}' \
 					"$_jl_scanid" "$_jl_state" "$_jl_pid" "$_jl_path" \
-					"${_meta_engine:--}" "$_jl_files" "$_jl_hits" "$_jl_elapsed"
+					"$_jl_engine" "$_jl_files" "$_jl_hits" "$_jl_elapsed" \
+					"$_jl_eta" "$_jl_workers" "$_jl_sig" \
+					"$_jl_prog_pos" "$_jl_prog_total"
 				;;
 		esac
 	done
@@ -743,20 +756,27 @@ _lmd_render_json_list() {
 			_lifecycle_read_meta "$_jl_stopped_sid" || continue
 			[ "$_first_stopped" != "1" ] && printf ","
 			_first_stopped=0
-			local _jl_sp="${_meta_path//\\/\\\\}"
-			_jl_sp="${_jl_sp//\"/\\\"}"
+			local _jl_sp _jl_sstage _jl_shr
+			_jl_sp=$(_json_escape_string "$_meta_path")
+			_jl_sstage=$(_json_escape_string "${_meta_stage:--}")
+			_jl_shr=$(_json_escape_string "${_meta_stopped_hr:-unknown}")
 			local _jl_sfiles="${_meta_total_files:-0}"; [ "$_jl_sfiles" = "-" ] && _jl_sfiles=0
 			local _jl_shits="${_meta_hits:-0}"; [ "$_jl_shits" = "-" ] && _jl_shits=0
-			printf '\n    {"scan_id": "%s", "stage": "%s", "total_files": %s, "hits": %s, "workers": "%s", "stopped_hr": "%s", "path": "%s"}' \
-				"$_jl_stopped_sid" "${_meta_stage:--}" "$_jl_sfiles" "$_jl_shits" \
-				"${_meta_workers:--}" "${_meta_stopped_hr:-unknown}" "$_jl_sp"
+			local _jl_selapsed="${_meta_elapsed:-0}"; [ "$_jl_selapsed" = "-" ] && _jl_selapsed=0
+			local _jl_sworkers="${_meta_workers:-0}"; [ "$_jl_sworkers" = "-" ] && _jl_sworkers=0
+			printf '\n    {"scan_id": "%s", "stage": "%s", "total_files": %s, "hits": %s, "elapsed": %s, "workers": %s, "stopped_hr": "%s", "path": "%s"}' \
+				"$_jl_stopped_sid" "$_jl_sstage" "$_jl_sfiles" "$_jl_shits" \
+				"$_jl_selapsed" "$_jl_sworkers" "$_jl_shr" "$_jl_sp"
 		fi
 	done
 
 	printf '\n  ],\n  "reports": ['
 	local _first=1
 	local _index_file="$sessdir/session.index"
-	local _seen_ids=""
+	# O(N) dedup between index and legacy passes; local -A is function-scoped
+	# (bash 4.0+) and does not leak. String concat + glob match was O(N²) and
+	# produced a visible hang on large indexes (~20K+ sessions).
+	local -A _seen_ids
 
 	# Rebuild index from TSV files if missing (first call on upgraded server)
 	if [ ! -f "$_index_file" ]; then
@@ -775,7 +795,7 @@ _lmd_render_json_list() {
 				_ix_path="$_ix_tot_quar"
 				_ix_tot_quar="0"
 			fi
-			_seen_ids="$_seen_ids $_ix_scanid"
+			_seen_ids["$_ix_scanid"]=1
 			if [ "$_first" != "1" ]; then printf ","; fi
 			_first=0
 			printf '\n    {'
@@ -792,8 +812,15 @@ _lmd_render_json_list() {
 			local _jquar="${_ix_tot_quar:-0}"
 			[ "$_jquar" = "-" ] && _jquar="0"
 			printf '"total_quarantined": %s, ' "$_jquar"
-			if [ "$_ix_elapsed" = "-" ]; then printf '"elapsed_seconds": null'
-			else printf '"elapsed_seconds": %s' "$_ix_elapsed"; fi
+			if [ "$_ix_elapsed" = "-" ]; then printf '"elapsed_seconds": null, '
+			else printf '"elapsed_seconds": %s, ' "$_ix_elapsed"; fi
+			if [ -z "$_ix_path" ] || [ "$_ix_path" = "-" ]; then
+				printf '"path": null'
+			else
+				# Out-param form avoids a subshell fork per report
+				_json_escape_var "$_ix_path"
+				printf '"path": "%s"' "$_JSON_ESC_OUT"
+			fi
 			printf '}'
 		done < "$_index_file"
 	fi
@@ -805,12 +832,12 @@ _lmd_render_json_list() {
 		[ -f "$_file" ] || continue
 		case "$_file" in *.tsv.*|*.hits.*) continue ;; esac
 		local _sid="${_file##*session.}"
-		case "$_seen_ids" in *" $_sid"*) continue ;; esac  # skip if already in index
+		[ -n "${_seen_ids[$_sid]:-}" ] && continue  # skip if already in index
 		# Clear vars before parsing (prevent stale data from prior iteration)
-		scanid="" scan_start_hr="" scan_end_hr="" scan_et="" tot_files="" tot_hits="" tot_cl=""
+		scanid="" scan_start_hr="" scan_end_hr="" scan_et="" tot_files="" tot_hits="" tot_cl="" hrspath=""
 		_parse_session_metadata "$_file"
 		[ -z "$scanid" ] && continue
-		_seen_ids="$_seen_ids $scanid"
+		_seen_ids["$scanid"]=1
 		if [ "$_first" != "1" ]; then printf ","; fi
 		_first=0
 		printf '\n    {'
@@ -827,6 +854,12 @@ _lmd_render_json_list() {
 		printf '"total_quarantined": null, '
 		if [ -z "$scan_et" ]; then printf '"elapsed_seconds": null, '
 		else printf '"elapsed_seconds": %s, ' "$scan_et"; fi
+		if [ -z "$hrspath" ]; then
+			printf '"path": null, '
+		else
+			_json_escape_var "$hrspath"
+			printf '"path": "%s", ' "$_JSON_ESC_OUT"
+		fi
 		printf '"source": "legacy"'
 		printf '}'
 	done
diff --git a/files/internals/lmd_hook.sh b/files/internals/lmd_hook.sh
@@ -16,18 +16,6 @@ _LMD_HOOK_LOADED=1
 # shellcheck disable=SC2034
 LMD_HOOK_VERSION="1.0.0"
 
-# _json_escape_string str — JSON-escape via bash param expansion (portable: $sed is "sed -E" on FreeBSD)
-# Backslash MUST be escaped first — invariant — else subsequent \t \r \n \" get re-escaped
-_json_escape_string() {
-	local _in="$1"
-	_in="${_in//\\/\\\\}"
-	_in="${_in//\"/\\\"}"
-	_in="${_in//$'\t'/\\t}"
-	_in="${_in//$'\r'/\\r}"
-	_in="${_in//$'\n'/\\n}"
-	printf '%s' "$_in"
-}
-
 # _scan_hook_validate hook_path — security validation (root-owned, not world-writable); returns 0/1
 _scan_hook_validate() {
 	local _hook_path="$1"
diff --git a/files/internals/lmd_lifecycle.sh b/files/internals/lmd_lifecycle.sh
@@ -384,6 +384,21 @@ _lifecycle_list_stopped() {
 	return 0
 }
 
+# _lifecycle_compute_eta engine elapsed prog_pos prog_total — echo ETA seconds, "null", or 0
+# clamav/clamdscan/yara → "null" (engines lack per-file progress data);
+# native with progress data → remaining seconds; otherwise "0".
+_lifecycle_compute_eta() {
+	local _engine="$1" _elapsed="$2" _pp="$3" _pt="$4"
+	case "$_engine" in
+		clamav|clamdscan|yara) printf 'null'; return 0 ;;
+	esac
+	if [ "$_pp" -gt 0 ] 2>/dev/null && [ "$_pt" -gt 0 ] 2>/dev/null && [ "$_elapsed" -gt 0 ] 2>/dev/null; then
+		printf '%d' "$(( (_elapsed * _pt / _pp) - _elapsed ))"
+	else
+		printf '0'
+	fi
+}
+
 # _lifecycle_render_json_active scanids — JSON array output (no jq dependency)
 _lifecycle_render_json_active() {
 	local _ids="$1"
@@ -402,15 +417,11 @@ _lifecycle_render_json_active() {
 			printf ',\n'
 		fi
 
-		# JSON-escape strings (backslash and double-quote)
-		local _j_path="${_meta_path//\\/\\\\}"
-		_j_path="${_j_path//\"/\\\"}"
-
-		local _j_stages="${_meta_stages//\\/\\\\}"
-		_j_stages="${_j_stages//\"/\\\"}"
-
-		local _j_sig="${_meta_sig_version//\\/\\\\}"
-		_j_sig="${_j_sig//\"/\\\"}"
+		local _j_path _j_stages _j_sig _j_engine
+		_j_path=$(_json_escape_string "$_meta_path")
+		_j_stages=$(_json_escape_string "${_meta_stages:-}")
+		_j_sig=$(_json_escape_string "${_meta_sig_version:-}")
+		_j_engine=$(_json_escape_string "${_meta_engine:--}")
 
 		local _i_pid="${_meta_pid:-0}"
 		local _i_total="${_meta_total_files:-0}"
@@ -433,26 +444,21 @@ _lifecycle_render_json_active() {
 		[ "$_i_prog_pos" = "-" ] && _i_prog_pos=0
 		[ "$_i_prog_total" = "-" ] && _i_prog_total=0
 
+		local _eta_val
+		_eta_val=$(_lifecycle_compute_eta "${_meta_engine:-}" "$_i_elapsed" "$_i_prog_pos" "$_i_prog_total")
+
 		printf '    {\n'
+		# scan_id: canonical field name (v2.0.1+); scanid retained for one release
+		# cycle for backward compat with existing consumers and removed in v2.1.0.
+		printf '      "scan_id": "%s",\n' "$_scanid"
 		printf '      "scanid": "%s",\n' "$_scanid"
 		printf '      "state": "%s",\n' "$_state"
 		printf '      "pid": %s,\n' "$_i_pid"
 		printf '      "path": "%s",\n' "$_j_path"
-		printf '      "engine": "%s",\n' "${_meta_engine:--}"
+		printf '      "engine": "%s",\n' "$_j_engine"
 		printf '      "total_files": %s,\n' "$_i_total"
 		printf '      "hits": %s,\n' "$_i_hits"
 		printf '      "elapsed": %s,\n' "$_i_elapsed"
-		# ETA: null for engines without per-file progress (clamav, yara);
-		# computed for native engine when progress data is available
-		local _eta_val="0"
-		case "${_meta_engine:-}" in
-			clamav|clamdscan|yara) _eta_val="null" ;;
-			*)
-				if [ "$_i_prog_pos" -gt 0 ] && [ "$_i_prog_total" -gt 0 ] && [ "$_i_elapsed" -gt 0 ]; then
-					_eta_val=$(( (_i_elapsed * _i_prog_total / _i_prog_pos) - _i_elapsed ))
-				fi
-				;;
-		esac
 		printf '      "eta": %s,\n' "$_eta_val"
 		printf '      "workers": %s,\n' "$_i_workers"
 		printf '      "stages": "%s",\n' "$_j_stages"
diff --git a/tests/31-json-report.bats b/tests/31-json-report.bats
diff --git a/tests/46-post-scan-hook.bats b/tests/46-post-scan-hook.bats
