History

fix: report captured service exec failures

Ben Davis committed 3 months ago

Commit b8328651 parent c53b2082 files · +196 −48Browse files

Showing the changes to guest (1 of 2 files). Show all

Unified · Split

guest/tx9-logs+122−48
@@ -12,6 +12,7 @@ import argparse · 11 unmodified lines
1212 import base64
1313 import ctypes
1414 import datetime as dt
15+import errno
1516 import hashlib
1617 import heapq
1718 import io
@@ -516,6 +517,14 @@ def nonnegative_float(value: str) -> float: · 498 unmodified lines
516517 return parsed
517518
518519
520+def write_exec_failure(status_fd: int, stage: str, error: int) -> None:
521+ payload = f"{stage}:{error}\n".encode("ascii")
522+ try:
523+ os.write(status_fd, payload)
524+ except OSError:
525+ pass
526+
527+
519528 def parent_death_exec(arguments: Sequence[str]) -> int:
520529 """Arm Linux parent-death SIGKILL, close the setup race, then exec.
521530
@@ -524,35 +533,40 @@ def parent_death_exec(arguments: Sequence[str]) -> int: · 2 unmodified lines
524533 outside this guarantee.
525534 """
526535
527− if len(arguments) < 3 or arguments[1] != "--":
536+ if len(arguments) < 4 or arguments[2] != "--":
528537 print("tx9-logs: invalid internal parent-death invocation", file=sys.stderr)
529538 return 126
530539 try:
531540 expected_parent = positive_int(arguments[0])
541+ status_fd = positive_int(arguments[1])
532542 except argparse.ArgumentTypeError as exc:
533− print(f"tx9-logs: invalid internal parent pid: {exc}", file=sys.stderr)
543+ print(f"tx9-logs: invalid internal parent-death argument: {exc}", file=sys.stderr)
534544 return 126
535− command = list(arguments[2:])
545+ command = list(arguments[3:])
536546 if not command:
537547 print("tx9-logs: internal parent-death exec requires a command", file=sys.stderr)
538548 return 126
539549
540− libc = ctypes.CDLL(None, use_errno=True)
541− prctl = libc.prctl
542− prctl.argtypes = [
543− ctypes.c_int,
544− ctypes.c_ulong,
545− ctypes.c_ulong,
546− ctypes.c_ulong,
547− ctypes.c_ulong,
548− ]
549− prctl.restype = ctypes.c_int
550− if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0:
551− error = ctypes.get_errno()
552− print(
553− f"tx9-logs: could not arm parent-death signal: {os.strerror(error)}",
554− file=sys.stderr,
555− )
550+ try:
551+ # Popen clears CLOEXEC for pass_fds so this fresh interpreter can
552+ # report setup/exec errors. Restore it before the target exec: EOF on
553+ # the parent end is then the unambiguous success handshake.
554+ os.set_inheritable(status_fd, False)
555+ libc = ctypes.CDLL(None, use_errno=True)
556+ prctl = libc.prctl
557+ prctl.argtypes = [
558+ ctypes.c_int,
559+ ctypes.c_ulong,
560+ ctypes.c_ulong,
561+ ctypes.c_ulong,
562+ ctypes.c_ulong,
563+ ]
564+ prctl.restype = ctypes.c_int
565+ if prctl(PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) != 0:
566+ error = ctypes.get_errno()
567+ raise OSError(error, os.strerror(error))
568+ except OSError as exc:
569+ write_exec_failure(status_fd, "setup", exc.errno or 0)
556570 return 126
557571
558572 # PR_SET_PDEATHSIG is not retroactive. If capture died between Popen's
@@ -564,7 +578,7 @@ def parent_death_exec(arguments: Sequence[str]) -> int: · 5 unmodified lines
564578 try:
565579 os.execvpe(command[0], command, os.environ)
566580 except OSError as exc:
567− print(f"tx9-logs: could not exec {Path(command[0]).name}: {exc}", file=sys.stderr)
581+ write_exec_failure(status_fd, "exec", exc.errno or 0)
568582 return 127 if isinstance(exc, FileNotFoundError) else 126
569583
570584
@@ -1087,59 +1101,119 @@ class Capture: · 516 unmodified lines
10871101 # subprocess.preexec_fn, whose Python callback is unsafe around the
10881102 # reader threads used by capture. Popen creates the new session first;
10891103 # the fresh interpreter then arms parent death and execs the command.
1090− launch_command = [
1091− sys.executable,
1092− str(Path(__file__).resolve()),
1093− PARENT_DEATH_EXEC_ACTION,
1094− str(os.getpid()),
1095− "--",
1096− *command,
1097− ]
1104+ status_read_fd = -1
1105+ status_write_fd = -1
1106+ reader_stop = threading.Event()
1107+ reader_cancelled = threading.Event()
1108+ readers: list[threading.Thread] = []
10981109 try:
1110+ status_read_fd, status_write_fd = os.pipe()
1111+ launch_command = [
1112+ sys.executable,
1113+ str(Path(__file__).resolve()),
1114+ PARENT_DEATH_EXEC_ACTION,
1115+ str(os.getpid()),
1116+ str(status_write_fd),
1117+ "--",
1118+ *command,
1119+ ]
10991120 child = subprocess.Popen(
11001121 launch_command,
11011122 stdout=subprocess.PIPE,
11021123 stderr=subprocess.PIPE,
11031124 env=child_env,
11041125 start_new_session=True,
1105− pass_fds=(self.mirror_out, self.mirror_err),
1126+ pass_fds=(self.mirror_out, self.mirror_err, status_write_fd),
11061127 )
11071128 self.child = child
11081129 self.active_pgid = child.pid
1130+ os.close(status_write_fd)
1131+ status_write_fd = -1
1132+ assert self.child.stdout is not None
1133+ assert self.child.stderr is not None
1134+ readers = [
1135+ threading.Thread(
1136+ target=self._reader,
1137+ args=("stdout", self.child.stdout, reader_stop, reader_cancelled),
1138+ daemon=True,
1139+ ),
1140+ threading.Thread(
1141+ target=self._reader,
1142+ args=("stderr", self.child.stderr, reader_stop, reader_cancelled),
1143+ daemon=True,
1144+ ),
1145+ ]
1146+ for reader in readers:
1147+ reader.start()
11091148 # A signal can arrive after Popen creates the process group but
11101149 # before the assignment above. Honor it before waiting so that
11111150 # startup cannot lose Docker's shutdown request.
11121151 if self.received_signal is not None:
11131152 self._forward_signal(self.received_signal, None)
1153+ read_fd = status_read_fd
1154+ status_read_fd = -1
1155+ with os.fdopen(read_fd, "rb", buffering=0) as status_pipe:
1156+ exec_status = status_pipe.read(128)
11141157 except OSError as exc:
1158+ if status_read_fd >= 0:
1159+ os.close(status_read_fd)
1160+ if status_write_fd >= 0:
1161+ os.close(status_write_fd)
1162+ if self.child is not None:
1163+ self._signal_process_group(signal.SIGKILL)
1164+ self.child.wait()
1165+ reader_stop.set()
1166+ for reader in readers:
1167+ reader.join()
1168+ if reader_cancelled.is_set():
1169+ self._signal_process_group(signal.SIGKILL)
1170+ self.active_pgid = None
1171+ self.child = None
1172+ self.flush_repeats()
11151173 message = f"could not start {Path(command[0]).name}: {exc}"
11161174 self.output("stderr", (message + "\n").encode())
11171175 self.structured("process_start_failed", message)
1118− return 127 if isinstance(exc, FileNotFoundError) else 126
1176+ status = 127 if isinstance(exc, FileNotFoundError) else 126
1177+ if self.received_signal is not None:
1178+ return 128 + self.received_signal
1179+ return status
1180+
1181+ if exec_status:
1182+ try:
1183+ stage, error_text = exec_status.decode("ascii").strip().split(":", 1)
1184+ error = int(error_text)
1185+ if stage not in {"setup", "exec"} or error < 0:
1186+ raise ValueError
1187+ except (UnicodeDecodeError, ValueError):
1188+ stage = "setup"
1189+ error = 0
1190+ self.child.wait()
1191+ reader_stop.set()
1192+ for reader in readers:
1193+ reader.join()
1194+ if reader_cancelled.is_set():
1195+ self._signal_process_group(signal.SIGKILL)
1196+ self.active_pgid = None
1197+ self.child = None
1198+ self.flush_repeats()
1199+ if stage == "exec":
1200+ detail = OSError(error, os.strerror(error), command[0])
1201+ status = 127 if error == errno.ENOENT else 126
1202+ else:
1203+ detail = OSError(error, os.strerror(error))
1204+ status = 126
1205+ message = f"could not start {Path(command[0]).name}: {detail}"
1206+ self.output("stderr", (message + "\n").encode())
1207+ self.structured("process_start_failed", message)
1208+ if self.received_signal is not None:
1209+ return 128 + self.received_signal
1210+ return status
11191211
11201212 self.structured(
11211213 "process_start",
11221214 f"{Path(command[0]).name} started",
11231215 {"pid": self.child.pid, "command": Path(command[0]).name},
11241216 )
1125− assert self.child.stdout is not None
1126− assert self.child.stderr is not None
1127− reader_stop = threading.Event()
1128− reader_cancelled = threading.Event()
1129− readers = [
1130− threading.Thread(
1131− target=self._reader,
1132− args=("stdout", self.child.stdout, reader_stop, reader_cancelled),
1133− daemon=True,
1134− ),
1135− threading.Thread(
1136− target=self._reader,
1137− args=("stderr", self.child.stderr, reader_stop, reader_cancelled),
1138− daemon=True,
1139− ),
1140− ]
1141− for reader in readers:
1142− reader.start()
11431217 return_code = self.child.wait()
11441218 # The direct child owns the capture lifecycle. Stop background
11451219 # descendants in its process group, then let readers drain buffered