[#396] FindPosmaster (linux) is rewritten

src/impl/platforms/linux/internal_platform_utils.py

@@ -9,9 +9,11 @@
 import re
 import shlex
 import typing
+import time
 
 
 class InternalPlatformUtils(base.InternalPlatformUtils):
+    C_MAX_FIND_POSTMASTER_ATTEMPTS = 5
     C_BASH_EXE = "/bin/bash"
 
     sm_exec_env = {
@@ -35,39 +37,106 @@ def FindPostmaster(
         assert len(bin_dir) > 0
         assert len(data_dir) > 0
 
+        failures: typing.List[Exception] = []
+
+        postmaster_pid: typing.Optional[int] = None
+
+        nAttempts = 0
+
+        while True:
+            nAttempts += 1
+
+            try:
+                postmaster_pid = __class__._FindPostmaster(
+                    os_ops,
+                    bin_dir,
+                    data_dir,
+                )
+            except Exception as e:
+                failures.append(e)
+
+                log_msg = "FindPostmaster (bin_dir={!r}, data_dir={!r}) detects a problem. Exception {}:\n{}".format(
+                    bin_dir,
+                    data_dir,
+                    type(e).__name__,
+                    e,
+                )
+                internal_utils.send_log_debug(log_msg)
+
+                if nAttempts < __class__.C_MAX_FIND_POSTMASTER_ATTEMPTS:
+                    time.sleep(0.05)
+                    continue
+
+                __class__._find_postmaster__throw_error__fail(
+                    bin_dir=bin_dir,
+                    data_dir=data_dir,
+                    failures=failures,
+                )
+
+            break
+
+        if postmaster_pid is None:
+            return InternalPlatformUtils.FindPostmasterResult.create_not_found()
+
+        assert type(postmaster_pid) is int
+
+        return InternalPlatformUtils.FindPostmasterResult.create_ok(postmaster_pid)
+
+    # --------------------------------------------------------------------
+    @staticmethod
+    def _FindPostmaster(
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str
+    ) -> typing.Optional[int]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) is str
+        assert type(data_dir) is str
+        assert type(__class__.C_BASH_EXE) is str
+        assert type(__class__.sm_exec_env) is dict
+        assert len(__class__.C_BASH_EXE) > 0
+        assert len(bin_dir) > 0
+        assert len(data_dir) > 0
+
         pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres"))
         data_dir_e = re.escape(data_dir)
 
         assert type(pg_path_e) is str
         assert type(data_dir_e) is str
 
-        regexp = r"^\s*[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?"
+        regexp = r"^\s*[0-9]+\s+[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?"
 
         cmd = [
             __class__.C_BASH_EXE,
             "-c",
-            "ps -ewwo \"pid=,args=\" | grep -E " + shlex.quote(regexp),
-        ]
+            "ps -ewwo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp),
+           ]
 
-        exit_status, output_b, error_b = os_ops.exec_command(
+        exec_r = os_ops.exec_command(
             cmd=cmd,
             ignore_errors=True,
             verbose=True,
             exec_env=__class__.sm_exec_env,
         )
 
+        assert type(exec_r) is tuple
+        assert len(exec_r) == 3
+
+        exit_status, output_b, error_b = exec_r
+
+        assert type(exit_status) is int
         assert type(output_b) is bytes
         assert type(error_b) is bytes
 
+        if exit_status == 1:
+            return None
+
         output = output_b.decode("utf-8")
         error = error_b.decode("utf-8")
 
         assert type(output) is str
         assert type(error) is str
 
-        if exit_status == 1:
-            return __class__.FindPostmasterResult.create_not_found()
-
         if exit_status != 0:
             errMsg = f"test command returned an unexpected exit code: {exit_status}"
             raise ExecUtilException(
@@ -82,43 +151,102 @@ def FindPostmaster(
         assert type(lines) is list
 
         if len(lines) == 0:
-            return __class__.FindPostmasterResult.create_not_found()
+            # ACHTUNG!
+            raise RuntimeError("Command returns 0 error code without output.")
+
+        # parse result lines
+        pid_to_ppid: __class__.T_PID_TO_PPID = {}
+
+        for i_line in range(len(lines)):
+            assert type(lines[i_line]) is str
+
+            parts = lines[i_line].split()
+            assert type(parts) is list
+
+            if len(parts) < 2:
+                __class__._find_postmaster__throw_error__bad_line_format(
+                    lines,
+                    i_line,
+                    "no usefull data",
+                )
+
+            if not parts[0].isdigit():
+                __class__._find_postmaster__throw_error__bad_line_format(
+                    lines,
+                    i_line,
+                    "bad pid",
+                )
+
+            if not parts[1].isdigit():
+                __class__._find_postmaster__throw_error__bad_line_format(
+                    lines,
+                    i_line,
+                    "bad ppid",
+                )
+
+            pid = int(parts[0])
+            ppid = int(parts[1])
+
+            if pid not in pid_to_ppid:
+                pid_to_ppid[pid] = ppid
+                continue
 
-        if len(lines) > 1:
-            msgs = []
-            msgs.append("Many processes like a postmaster are found: {}.".format(len(lines)))
+            other_ppid = pid_to_ppid[pid]
+            assert type(other_ppid) is int
 
-            for i in range(len(lines)):
-                assert type(lines[i]) is str
-                lines.append("[{}] '{}'".format(i, lines[i]))
+            if ppid == other_ppid:
+                log_msg = "FindPostmaster (data_dir={!r}) get pid ({}) with ppid ({}) more than one time.".format(
+                    data_dir,
+                    pid,
+                    ppid,
+                )
+                internal_utils.send_log_debug(log_msg)
                 continue
 
-            internal_utils.send_log_debug("\n".join(lines))
-            return __class__.FindPostmasterResult.create_many_processes()
+            # ACTUNG ppid is changed --> restart
+            __class__._find_postmaster__throw_error__ppid_is_changed(
+                pid,
+                ppid,
+                other_ppid,
+                lines,
+            )
+
+        assert len(pid_to_ppid) <= len(lines)
 
-        def is_space_or_tab(ch) -> bool:
-            assert type(ch) is str
-            return ch == " " or ch == "\t"
+        true_postmasters = [
+            pid for pid, ppid in pid_to_ppid.items()
+            if ppid not in pid_to_ppid
+        ]
 
-        line = lines[0]
-        start = 0
-        while start < len(line) and is_space_or_tab(line[start]):
-            start += 1
+        if len(true_postmasters) == 0:
+            __class__._find_postmaster__throw_error__cycle(
+                pid_to_ppid,
+            )
 
-        pos = start
-        while pos < len(line) and line[pos].isnumeric():
-            pos += 1
+        if len(true_postmasters) == 1:
+            true_pid = true_postmasters[0]
 
-        if pos == start:
-            return __class__.FindPostmasterResult.create_has_problems()
+            if len(pid_to_ppid) > 1:
+                msg = "Many processes like a postmaster for data dir [{}] are found ({}).".format(
+                    data_dir,
+                    len(true_postmasters),
+                )
 
-        if pos != len(line) and not line[pos].isspace():
-            return __class__.FindPostmasterResult.create_has_problems()
+                msg += " List (ppid->pid): {}.".format(
+                    __class__._make_text_from_pid_to_ppid(pid_to_ppid),
+                )
 
-        pid = int(line[start:pos])
-        assert type(pid) is int
+                msg += " True postmaster PID is {}.".format(true_pid)
+                internal_utils.send_log_debug(msg)
 
-        return __class__.FindPostmasterResult.create_ok(pid)
+            return true_pid
+
+        assert len(true_postmasters) > 1
+
+        __class__._find_postmaster__throw_error__many_postmasters(
+            true_postmasters,
+            pid_to_ppid,
+        )
 
     def ProcessIsZombi_soft_check(
         self,
@@ -169,3 +297,141 @@ def _is_file_not_found_exception(e: Exception) -> bool:
                 return True
 
         return False
+
+    T_PID_TO_PPID = typing.Dict[int, int]
+
+    @staticmethod
+    def _make_text_from_pid_to_ppid(pid_to_ppid: T_PID_TO_PPID) -> str:
+        result = ""
+        sep = ""
+        for pid, ppid in pid_to_ppid.items():
+            result += sep + " {}->{}".format(ppid, pid)
+            sep = ", "
+            continue
+        return result
+
+    @staticmethod
+    def _find_postmaster__throw_error__bad_line_format(
+        lines: typing.List[str],
+        i_line: int,
+        hint: str,
+    ) -> typing.NoReturn:
+        assert type(lines) is list
+        assert type(i_line) is int
+        assert type(hint) is int
+
+        error_lines: typing.List[str] = []
+        error_lines.append(
+            "Line {} has bad format. Hint: {}.".format(
+                i_line + 1,
+                hint,
+            ),
+        )
+        error_lines.append(
+            "Problem line is:"
+        )
+        error_lines.append(
+            " " + repr(lines[i_line]),
+        )
+        error_lines.append(
+            "All the lines is:",
+        )
+        for i in range(len(lines)):
+            error_lines.append(
+                " {}. {!r}".format(i + 1, lines[i]),
+            )
+            continue
+
+        raise RuntimeError("\n".join(error_lines))
+
+    @staticmethod
+    def _find_postmaster__throw_error__ppid_is_changed(
+        pid: int,
+        ppid: int,
+        other_ppid: int,
+        lines: typing.List[str],
+    ) -> typing.NoReturn:
+        assert type(pid) is int
+        assert type(ppid) is int
+        assert type(other_ppid) is int
+        assert type(lines) is list
+
+        error_lines: typing.List[str] = []
+        error_lines.append(
+            "Parent of process ({}) is changed from {} to {}.".format(
+                pid,
+                other_ppid,
+                ppid,
+            ),
+        )
+        error_lines.append(
+            "All the lines is:",
+        )
+        for i in range(len(lines)):
+            error_lines.append(
+                " {}. {!r}".format(i + 1, lines[i]),
+            )
+            continue
+
+        raise RuntimeError("\n".join(error_lines))
+
+    @staticmethod
+    def _find_postmaster__throw_error__cycle(
+        pid_to_ppid: T_PID_TO_PPID,
+    ) -> typing.NoReturn:
+        msg = "Cycle in processes postgres process tree. "
+
+        msg += " List (ppid->pid): {},".format(
+            __class__._make_text_from_pid_to_ppid(pid_to_ppid),
+        )
+
+        msg += " List size is {}.".format(
+            len(pid_to_ppid),
+        )
+        raise RuntimeError(msg)
+
+    @staticmethod
+    def _find_postmaster__throw_error__many_postmasters(
+        postmaster_pids: typing.List[int],
+        pid_to_ppid: T_PID_TO_PPID,
+    ) -> typing.NoReturn:
+        msg = "Many processes like a postmaster are found ({}): {}.".format(
+            len(postmaster_pids),
+            ", ".join(map(str, postmaster_pids)),
+        )
+
+        msg += " Trees (ppid->pid): {}.".format(
+            __class__._make_text_from_pid_to_ppid(pid_to_ppid),
+        )
+
+        msg += " Total process count is {}.".format(len(pid_to_ppid))
+        raise RuntimeError(msg)
+
+    @staticmethod
+    def _find_postmaster__throw_error__fail(
+        bin_dir: str,
+        data_dir: str,
+        failures: typing.List[Exception],
+    ) -> typing.NoReturn:
+        assert type(bin_dir) is str
+        assert type(data_dir) is str
+        assert type(failures) is list
+
+        err_msg = "FindPostmater did {} attempts and has not gotten a stable result. List of failures:\n"
+
+        n = 0
+        sep = ""
+        for e in failures:
+            assert isinstance(e, Exception)
+
+            n += 1
+            err_msg += sep
+            err_msg += "Failure #{}. Exception ({}):\n{}".format(
+                n,
+                type(e).__name__,
+                str(e),
+            )
+            sep = "\n"
+            continue
+
+        raise RuntimeError(err_msg)