build_manager: track time_started and handle timeout state

build_manager/manager.py

@@ -15,6 +15,7 @@ class BuildState(Enum):
     SUCCESS = 2
     FAILURE = 3
     ERROR = 4
+    TIMED_OUT = 5
 
 
 class BuildProgress:
@@ -71,6 +72,7 @@ class BuildInfo:
             percent=0
         )
         self.time_created = time.time()
+        self.time_started = None # when build state becomes RUNNING
 
     def to_dict(self) -> dict:
         return {
@@ -81,6 +83,7 @@ class BuildInfo:
             'selected_features': list(self.selected_features),
             'progress': self.progress.to_dict(),
             'time_created': self.time_created,
+            'time_started': getattr(self, 'time_started', None),
         }
 
 
@@ -353,6 +356,27 @@ class BuildManager:
             keepttl=True
         )
 
+    def update_build_time_started(self,
+                              build_id: str,
+                              time_started: float) -> None:
+        """
+        Update the build's time_started timestamp.
+
+        Parameters:
+            build_id (str): The ID of the build to update.
+            time_started (float): The timestamp when the build started running.
+        """
+        build_info = self.get_build_info(build_id=build_id)
+
+        if build_info is None:
+            raise ValueError(f"Build with id {build_id} not found.")
+
+        build_info.time_started = time_started
+        self.__update_build_info(
+            build_id=build_id,
+            build_info=build_info
+        )
+
     def update_build_progress_percent(self,
                                       build_id: str,
                                       percent: int) -> None:

build_manager/progress_updater.py

@@ -6,7 +6,9 @@ from .manager import (
     BuildManager as bm,
     BuildState
 )
+import time
 
+CBS_BUILD_TIMEOUT_SEC = int(os.getenv('CBS_BUILD_TIMEOUT_SEC', 900))  # 15 minutes default
 
 class BuildProgressUpdater:
     """
@@ -157,6 +159,28 @@ class BuildProgressUpdater:
             raise RuntimeError(
                 "This method should only be called for running builds."
             )
+        # Set time_started if not already set
+        if build_info.time_started is None:
+            start_time = time.time()
+            bm.get_singleton().update_build_time_started(
+                build_id=build_id,
+                time_started=start_time
+            )
+            self.logger.info(
+                f"Build {build_id} started running at {start_time}"
+            )
+            build_info.time_started = start_time
+
+        # Check for timeout
+        elapsed = time.time() - build_info.time_started
+        if elapsed > CBS_BUILD_TIMEOUT_SEC:
+            self.logger.warning(
+                f"Build {build_id} timed out after {elapsed:.0f} seconds"
+            )
+            build_info.error_message = (
+                f"Build exceeded {CBS_BUILD_TIMEOUT_SEC // 60} minute timeout"
+            )
+            return BuildState.TIMED_OUT
 
         # Builder ships the archive post completion
         # This is irrespective of SUCCESS or FAILURE
@@ -213,6 +237,9 @@ class BuildProgressUpdater:
         elif current_state == BuildState.ERROR:
             # Keep existing percentage
             pass
+        elif current_state == BuildState.TIMED_OUT:
+            # Keep existing percentage
+            pass
         else:
             raise Exception("Unhandled BuildState.")
 
@@ -259,6 +286,9 @@ class BuildProgressUpdater:
         elif current_state == BuildState.ERROR:
             # ERROR is a conclusive state
             pass
+        elif current_state == BuildState.TIMED_OUT:
+            # TIMED_OUT is a conclusive state
+            pass
         else:
             raise Exception("Unhandled BuildState.")