From a4c5b5a21ee550104defe72a8417595121511607 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 13 Jul 2026 13:24:10 +0300 Subject: [PATCH] [#396] FindPosmaster (linux) is rewritten New code: - builds a tree - detects cycles - detects multple trees - does 5 attempts - logs all failures - returns ok/not_found or raises an exception Test is added. --- .../linux/internal_platform_utils.py | 332 ++++++++++++++++-- .../test_set010__FindPostmaster.py | 104 ++++++ 2 files changed, 403 insertions(+), 33 deletions(-) create mode 100755 tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py diff --git a/src/impl/platforms/linux/internal_platform_utils.py b/src/impl/platforms/linux/internal_platform_utils.py index b0ed64c3..3834f6e7 100644 --- a/src/impl/platforms/linux/internal_platform_utils.py +++ b/src/impl/platforms/linux/internal_platform_utils.py @@ -9,9 +9,11 @@ import re import shlex import typing +import time class InternalPlatformUtils(base.InternalPlatformUtils): + C_MAX_FIND_POSTMASTER_ATTEMPTS = 5 C_BASH_EXE = "/bin/bash" sm_exec_env = { @@ -35,39 +37,106 @@ def FindPostmaster( assert len(bin_dir) > 0 assert len(data_dir) > 0 + failures: typing.List[Exception] = [] + + postmaster_pid: typing.Optional[int] = None + + nAttempts = 0 + + while True: + nAttempts += 1 + + try: + postmaster_pid = __class__._FindPostmaster( + os_ops, + bin_dir, + data_dir, + ) + except Exception as e: + failures.append(e) + + log_msg = "FindPostmaster (bin_dir={!r}, data_dir={!r}) detects a problem. Exception {}:\n{}".format( + bin_dir, + data_dir, + type(e).__name__, + e, + ) + internal_utils.send_log_debug(log_msg) + + if nAttempts < __class__.C_MAX_FIND_POSTMASTER_ATTEMPTS: + time.sleep(0.05) + continue + + __class__._find_postmaster__throw_error__fail( + bin_dir=bin_dir, + data_dir=data_dir, + failures=failures, + ) + + break + + if postmaster_pid is None: + return InternalPlatformUtils.FindPostmasterResult.create_not_found() + + assert type(postmaster_pid) is int + + return InternalPlatformUtils.FindPostmasterResult.create_ok(postmaster_pid) + + # -------------------------------------------------------------------- + @staticmethod + def _FindPostmaster( + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> typing.Optional[int]: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(__class__.C_BASH_EXE) is str + assert type(__class__.sm_exec_env) is dict + assert len(__class__.C_BASH_EXE) > 0 + assert len(bin_dir) > 0 + assert len(data_dir) > 0 + pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres")) data_dir_e = re.escape(data_dir) assert type(pg_path_e) is str assert type(data_dir_e) is str - regexp = r"^\s*[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?" + regexp = r"^\s*[0-9]+\s+[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?" cmd = [ __class__.C_BASH_EXE, "-c", - "ps -ewwo \"pid=,args=\" | grep -E " + shlex.quote(regexp), - ] + "ps -ewwo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp), + ] - exit_status, output_b, error_b = os_ops.exec_command( + exec_r = os_ops.exec_command( cmd=cmd, ignore_errors=True, verbose=True, exec_env=__class__.sm_exec_env, ) + assert type(exec_r) is tuple + assert len(exec_r) == 3 + + exit_status, output_b, error_b = exec_r + + assert type(exit_status) is int assert type(output_b) is bytes assert type(error_b) is bytes + if exit_status == 1: + return None + output = output_b.decode("utf-8") error = error_b.decode("utf-8") assert type(output) is str assert type(error) is str - if exit_status == 1: - return __class__.FindPostmasterResult.create_not_found() - if exit_status != 0: errMsg = f"test command returned an unexpected exit code: {exit_status}" raise ExecUtilException( @@ -82,43 +151,102 @@ def FindPostmaster( assert type(lines) is list if len(lines) == 0: - return __class__.FindPostmasterResult.create_not_found() + # ACHTUNG! + raise RuntimeError("Command returns 0 error code without output.") + + # parse result lines + pid_to_ppid: __class__.T_PID_TO_PPID = {} + + for i_line in range(len(lines)): + assert type(lines[i_line]) is str + + parts = lines[i_line].split() + assert type(parts) is list + + if len(parts) < 2: + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "no usefull data", + ) + + if not parts[0].isdigit(): + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "bad pid", + ) + + if not parts[1].isdigit(): + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "bad ppid", + ) + + pid = int(parts[0]) + ppid = int(parts[1]) + + if pid not in pid_to_ppid: + pid_to_ppid[pid] = ppid + continue - if len(lines) > 1: - msgs = [] - msgs.append("Many processes like a postmaster are found: {}.".format(len(lines))) + other_ppid = pid_to_ppid[pid] + assert type(other_ppid) is int - for i in range(len(lines)): - assert type(lines[i]) is str - lines.append("[{}] '{}'".format(i, lines[i])) + if ppid == other_ppid: + log_msg = "FindPostmaster (data_dir={!r}) get pid ({}) with ppid ({}) more than one time.".format( + data_dir, + pid, + ppid, + ) + internal_utils.send_log_debug(log_msg) continue - internal_utils.send_log_debug("\n".join(lines)) - return __class__.FindPostmasterResult.create_many_processes() + # ACTUNG ppid is changed --> restart + __class__._find_postmaster__throw_error__ppid_is_changed( + pid, + ppid, + other_ppid, + lines, + ) + + assert len(pid_to_ppid) <= len(lines) - def is_space_or_tab(ch) -> bool: - assert type(ch) is str - return ch == " " or ch == "\t" + true_postmasters = [ + pid for pid, ppid in pid_to_ppid.items() + if ppid not in pid_to_ppid + ] - line = lines[0] - start = 0 - while start < len(line) and is_space_or_tab(line[start]): - start += 1 + if len(true_postmasters) == 0: + __class__._find_postmaster__throw_error__cycle( + pid_to_ppid, + ) - pos = start - while pos < len(line) and line[pos].isnumeric(): - pos += 1 + if len(true_postmasters) == 1: + true_pid = true_postmasters[0] - if pos == start: - return __class__.FindPostmasterResult.create_has_problems() + if len(pid_to_ppid) > 1: + msg = "Many processes like a postmaster for data dir [{}] are found ({}).".format( + data_dir, + len(true_postmasters), + ) - if pos != len(line) and not line[pos].isspace(): - return __class__.FindPostmasterResult.create_has_problems() + msg += " List (ppid->pid): {}.".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) - pid = int(line[start:pos]) - assert type(pid) is int + msg += " True postmaster PID is {}.".format(true_pid) + internal_utils.send_log_debug(msg) - return __class__.FindPostmasterResult.create_ok(pid) + return true_pid + + assert len(true_postmasters) > 1 + + __class__._find_postmaster__throw_error__many_postmasters( + true_postmasters, + pid_to_ppid, + ) def ProcessIsZombi_soft_check( self, @@ -169,3 +297,141 @@ def _is_file_not_found_exception(e: Exception) -> bool: return True return False + + T_PID_TO_PPID = typing.Dict[int, int] + + @staticmethod + def _make_text_from_pid_to_ppid(pid_to_ppid: T_PID_TO_PPID) -> str: + result = "" + sep = "" + for pid, ppid in pid_to_ppid.items(): + result += sep + " {}->{}".format(ppid, pid) + sep = ", " + continue + return result + + @staticmethod + def _find_postmaster__throw_error__bad_line_format( + lines: typing.List[str], + i_line: int, + hint: str, + ) -> typing.NoReturn: + assert type(lines) is list + assert type(i_line) is int + assert type(hint) is int + + error_lines: typing.List[str] = [] + error_lines.append( + "Line {} has bad format. Hint: {}.".format( + i_line + 1, + hint, + ), + ) + error_lines.append( + "Problem line is:" + ) + error_lines.append( + " " + repr(lines[i_line]), + ) + error_lines.append( + "All the lines is:", + ) + for i in range(len(lines)): + error_lines.append( + " {}. {!r}".format(i + 1, lines[i]), + ) + continue + + raise RuntimeError("\n".join(error_lines)) + + @staticmethod + def _find_postmaster__throw_error__ppid_is_changed( + pid: int, + ppid: int, + other_ppid: int, + lines: typing.List[str], + ) -> typing.NoReturn: + assert type(pid) is int + assert type(ppid) is int + assert type(other_ppid) is int + assert type(lines) is list + + error_lines: typing.List[str] = [] + error_lines.append( + "Parent of process ({}) is changed from {} to {}.".format( + pid, + other_ppid, + ppid, + ), + ) + error_lines.append( + "All the lines is:", + ) + for i in range(len(lines)): + error_lines.append( + " {}. {!r}".format(i + 1, lines[i]), + ) + continue + + raise RuntimeError("\n".join(error_lines)) + + @staticmethod + def _find_postmaster__throw_error__cycle( + pid_to_ppid: T_PID_TO_PPID, + ) -> typing.NoReturn: + msg = "Cycle in processes postgres process tree. " + + msg += " List (ppid->pid): {},".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " List size is {}.".format( + len(pid_to_ppid), + ) + raise RuntimeError(msg) + + @staticmethod + def _find_postmaster__throw_error__many_postmasters( + postmaster_pids: typing.List[int], + pid_to_ppid: T_PID_TO_PPID, + ) -> typing.NoReturn: + msg = "Many processes like a postmaster are found ({}): {}.".format( + len(postmaster_pids), + ", ".join(map(str, postmaster_pids)), + ) + + msg += " Trees (ppid->pid): {}.".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " Total process count is {}.".format(len(pid_to_ppid)) + raise RuntimeError(msg) + + @staticmethod + def _find_postmaster__throw_error__fail( + bin_dir: str, + data_dir: str, + failures: typing.List[Exception], + ) -> typing.NoReturn: + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(failures) is list + + err_msg = "FindPostmater did {} attempts and has not gotten a stable result. List of failures:\n" + + n = 0 + sep = "" + for e in failures: + assert isinstance(e, Exception) + + n += 1 + err_msg += sep + err_msg += "Failure #{}. Exception ({}):\n{}".format( + n, + type(e).__name__, + str(e), + ) + sep = "\n" + continue + + raise RuntimeError(err_msg) diff --git a/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py b/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py new file mode 100755 index 00000000..4696cdff --- /dev/null +++ b/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from tests.helpers.global_data import PostgresNodeService +from tests.helpers.global_data import PostgresNodeServices +from tests.helpers.global_data import OsOperations +from tests.helpers.global_data import PortManager +from tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils + +from src import PostgresNode +from src import NodeStatus +from src.impl.platforms.internal_platform_utils_factory import create_internal_platform_utils +from src.impl.platforms.internal_platform_utils_factory import InternalPlatformUtils + +import pytest +import typing + + +class TestSet010__FindPostmaster: + @pytest.fixture( + params=PostgresNodeServices.sm_locals_and_remotes, + ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes] + ) + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) + return request.param + + class tagData001: + wait: typing.Optional[bool] + + def __init__(self, wait: typing.Optional[bool]): + assert wait is None or type(wait) is bool + self.wait = wait + return + + def test_001__ok( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node.start() + assert node.is_started + assert node.status() == NodeStatus.Running + + # Internals + assert type(node._manually_started_pm_pid) is int + assert node._manually_started_pm_pid != 0 + assert node._manually_started_pm_pid != node._C_PM_PID__IS_NOT_DETECTED + assert node._manually_started_pm_pid == node.pid + + platform_utils = create_internal_platform_utils(node.os_ops) + assert platform_utils is not None + assert isinstance(platform_utils, InternalPlatformUtils) + + r = platform_utils.FindPostmaster( + node.os_ops, + node.bin_dir, + node.data_dir + ) + + assert r is not None + assert type(r) is InternalPlatformUtils.FindPostmasterResult + assert r.code == InternalPlatformUtils.FindPostmasterResultCode.ok + assert r.pid == node.pid + + return + + def test_002__not_found( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + platform_utils = create_internal_platform_utils(node.os_ops) + assert platform_utils is not None + assert isinstance(platform_utils, InternalPlatformUtils) + + r = platform_utils.FindPostmaster( + node.os_ops, + node.bin_dir, + node.data_dir + ) + + assert r is not None + assert type(r) is InternalPlatformUtils.FindPostmasterResult + assert r.code == InternalPlatformUtils.FindPostmasterResultCode.not_found + assert r.pid is None + + return