Only remove ‘au’ nodes after boot_timeout

Author: Ryan Hamel

gridengine/src/gridengine/autoscaler.py

@@ -1,13 +1,15 @@
 import os
 import sys
+import time
 import typing
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Callable, Dict, List, Optional, Set
 
 from hpc.autoscale import hpclogging as logging
 from hpc.autoscale.job import demandcalculator as dcalclib
 from hpc.autoscale.job import demandprinter
 from hpc.autoscale.job.demand import DemandResult
 from hpc.autoscale.job.demandcalculator import DemandCalculator
+from hpc.autoscale.node.node import Node
 from hpc.autoscale.node.nodehistory import NodeHistory, SQLiteNodeHistory
 from hpc.autoscale.node.nodemanager import new_node_manager
 from hpc.autoscale.results import DefaultContextHandler
@@ -30,6 +32,7 @@ def autoscale_grid_engine(
     ctx_handler: Optional[DefaultContextHandler] = None,
     node_history: Optional[NodeHistory] = None,
     dry_run: bool = False,
+    clock: Callable[[], float] = time.time,
 ) -> DemandResult:
     global _exit_code
 
@@ -75,8 +78,11 @@ def autoscale_grid_engine(
             if not ccnodeid or ccnodeid == cc_node.delayed_node_id.node_id:
                 if cc_node.state in ["Preparing", "Acquiring"]:
                     continue
+
+        # Sometimes a node will be fully converged but will still be shown as 'au' = alarm, unresponsive
+        # We will ignore this unless the node has been given its full boot_timeout.
         if "a" in state and "u" in state:
-            invalid_nodes.append(node)
+            _handle_alarm_node(config, clock, node, invalid_nodes)
 
     # nodes in error state must also be deleted
     nodes_to_delete = ge_driver.clean_hosts(invalid_nodes)
@@ -168,6 +174,25 @@ def autoscale_grid_engine(
     return demand_result
 
 
+def _handle_alarm_node(
+    config: Dict,
+    clock: Callable[[], float],
+    node: "Node",
+    invalid_nodes_out: List["Node"],
+) -> None:
+    now = clock()
+    omega = config["boot_timeout"] + node.create_time_unix
+    if now > omega:
+        invalid_nodes_out.append(node)
+    else:
+        logging.warning(
+            f"Node {node.name} is in an alarm state ('au') but it has not been up longer than its boot_timeout."
+        )
+        logging.warning(
+            f"If {node.name} is still in an alarm state in {omega - now} seconds, it will be removed and shutdown."
+        )
+
+
 def new_demand_calculator(
     config: Dict,
     ge_env: Optional[GridEngineEnvironment] = None,

gridengine/test/gridengine_test/autoscaler_test.py

@@ -1,5 +1,5 @@
 import os
-from typing import Dict, List, Optional
+from typing import Callable, Dict, List, Optional
 
 from hpc.autoscale import hpclogging
 from hpc.autoscale.ccbindings.mock import MockClusterBinding
@@ -314,6 +314,31 @@ def test_fill_up_and_round_robin() -> None:
     )
 
 
+def test_handle_alarm_node() -> None:
+    class MockNode:
+        def __init__(self, name: str, create_time_unix: float) -> None:
+            self.name = name
+            self.create_time_unix = create_time_unix
+
+        def __str__(self) -> str:
+            return f"Node({self.name})"
+
+    def make_clock(t: float) -> Callable[[], float]:
+        return lambda: t
+
+    node = MockNode("htc-1", 100)
+    config = {"boot_timeout": 1800}
+
+    def run_test(now: float, invalid: bool) -> None:
+        invalid_nodes_out = []
+        autoscaler._handle_alarm_node(config, make_clock(now), node, invalid_nodes_out)
+        assert invalid == bool(invalid_nodes_out)
+
+    run_test(100, False)
+    run_test(1899, False)
+    run_test(1901, True)
+
+
 def _job(qsub_cmd: str, job_id: int) -> Job:
     ge_env = common_ge_env()
     qsub = mock_driver.MockQsub(ge_env)
@@ -360,7 +385,7 @@ def common_cluster_test(
     qsub_commands: List[str],
     pg_counts: Optional[Dict[str, int]] = None,
     previous_dcalc: Optional[DemandCalculator] = None,
-    **array_counts: int
+    **array_counts: int,
 ) -> DemandCalculator:
     pg_counts = pg_counts or {}
     dcalc = common_cluster(qsub_commands, previous_dcalc)