web: use new provider based versions manager

web/api/v1/admin.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, HTTPException, Depends, status
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 
-from web.schemas import RefreshRemotesResponse
+from web.schemas import RefreshVersionsResponse
 from web.services.admin import get_admin_service, AdminService
 
 
@@ -9,12 +9,12 @@ router = APIRouter(prefix="/admin", tags=["admin"])
 security = HTTPBearer(auto_error=False)
 
 
-async def verify_remote_reload_token(
+async def verify_admin_token(
     credentials: HTTPAuthorizationCredentials = Depends(security),
     admin_service: AdminService = Depends(get_admin_service)
 ) -> None:
     """
-    Verify the bearer token for remote reload authentication.
+    Verify the bearer token for admin API authentication.
 
     Args:
         credentials: HTTP authorization credentials from request header
@@ -32,7 +32,7 @@ async def verify_remote_reload_token(
 
     token = credentials.credentials
     try:
-        if not await admin_service.verify_remote_reload_token(token):
+        if not await admin_service.verify_admin_token(token):
             raise HTTPException(
                 status_code=status.HTTP_401_UNAUTHORIZED,
                 detail="Invalid authentication token"
@@ -45,8 +45,8 @@ async def verify_remote_reload_token(
 
 
 @router.post(
-    "/refresh_remotes",
-    response_model=RefreshRemotesResponse,
+    "/refresh_versions",
+    response_model=RefreshVersionsResponse,
     responses={
         401: {"description": "Invalid or missing authentication token"},
         500: {
@@ -57,12 +57,12 @@ async def verify_remote_reload_token(
         }
     }
 )
-async def refresh_remotes(
-    _: None = Depends(verify_remote_reload_token),
+async def refresh_versions(
+    _: None = Depends(verify_admin_token),
     admin_service: AdminService = Depends(get_admin_service)
 ):
     """
-    Trigger a hot reset/refresh of remote metadata.
+    Trigger a refresh of all version metadata providers.
 
     This endpoint requires bearer token authentication in the Authorization
     header:
@@ -71,17 +71,17 @@ async def refresh_remotes(
     ```
 
     Returns:
-        RefreshRemotesResponse: List of remotes that were refreshed
+        RefreshVersionsResponse: Git remotes synced after the refresh
 
     Raises:
         401: Invalid or missing authentication token
         500: Refresh operation failed
     """
     try:
-        remotes = await admin_service.refresh_remotes()
-        return RefreshRemotesResponse(remotes=remotes)
+        remotes = await admin_service.refresh_versions()
+        return RefreshVersionsResponse(remotes=remotes)
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to refresh remotes: {str(e)}"
+            detail=f"Failed to refresh versions: {str(e)}"
         )

web/core/config.py

@@ -88,33 +88,43 @@ class Settings:
         return os.getenv('CBS_ENABLE_INBUILT_BUILDER', '1') == '1'
 
     @property
-    def remote_reload_token(self) -> Optional[str]:
+    def manifest_cache_dir(self) -> str:
+        """Directory for cached firmware manifest files."""
+        return os.path.dirname(self.remotes_json_path)
+
+    @property
+    def ap_firmware_manifest_url(self) -> str:
+        """URL for the ArduPilot firmware manifest."""
+        return os.getenv(
+            'CBS_AP_FIRMWARE_MANIFEST_URL',
+            'https://firmware.ardupilot.org/manifest.json',
+        )
+
+    @property
+    def admin_token(self) -> Optional[str]:
         """
-        Get remote reload token from file or environment variable.
+        Get admin API token from file or environment variable.
 
         Tries to read token from file first, falls back to environment variable.
 
         Returns:
             The authorization token if found, None otherwise
         """
-        token_file_path = os.path.join(self.base_dir, 'secrets', 'reload_token')
+        token_file_path = os.path.join(self.base_dir, 'secrets', 'admin_token')
 
         try:
-            # Try to read the secret token from the file
             with open(token_file_path, 'r') as file:
                 token = file.read().strip()
                 return token
         except (FileNotFoundError, PermissionError):
-            # If the file does not exist or no permission, check environment
-            env_token = os.getenv('CBS_REMOTES_RELOAD_TOKEN', '')
+            env_token = os.getenv('CBS_ADMIN_TOKEN', '')
             return env_token if env_token != '' else None
         except Exception as e:
             logger.error(
                 f"Unexpected error reading token file at {token_file_path}: {e}. "
                 "Checking environment for token."
             )
-            # For any other error, fall back to environment variable
-            env_token = os.getenv('CBS_REMOTES_RELOAD_TOKEN', None)
+            env_token = os.getenv('CBS_ADMIN_TOKEN', None)
             return env_token if env_token != '' else None
 
 

web/core/startup.py

@@ -25,7 +25,6 @@ def ensure_base_structure(base_dir: str) -> None:
         logger.warning("Base directory not specified, skipping initialization")
         return
 
-    # Define required subdirectories
     subdirs = [
         'artifacts',
         'configs',
@@ -39,52 +38,12 @@ def ensure_base_structure(base_dir: str) -> None:
         logger.debug(f"Ensured directory exists: {path}")
 
 
-def ensure_remotes_json(base_dir: str, remote_name: str = "ardupilot") -> None:
-    """
-    Ensure remotes.json configuration file exists.
-
-    If the remotes.json file doesn't exist, creates it by fetching release
-    information from the specified remote.
-
-    Args:
-        base_dir: The base directory path (typically from CBS_BASEDIR)
-        remote_name: The remote repository name to fetch releases from
-    """
-    if not base_dir:
-        logger.warning(
-            "Base directory not specified, "
-            "skipping remotes.json initialization"
-        )
-        return
-
-    remotes_json_path = os.path.join(base_dir, 'configs', 'remotes.json')
-
-    if not os.path.isfile(remotes_json_path):
-        logger.info(
-            f"remotes.json not found at {remotes_json_path}, "
-            f"creating it..."
-        )
-        try:
-            from scripts import fetch_releases
-            fetch_releases.run(
-                base_dir=base_dir,
-                remote_name=remote_name,
-            )
-            logger.info("Successfully created remotes.json")
-        except Exception as e:
-            logger.error(f"Failed to create remotes.json: {e}")
-            raise
-    else:
-        logger.debug(f"remotes.json already exists at {remotes_json_path}")
-
-
 def initialize_application(base_dir: str) -> None:
     """
     Initialize the application environment.
 
-    Performs all necessary setup operations including:
-    - Creating required directory structure
-    - Ensuring remotes.json configuration exists
+    Performs all necessary setup operations including creating the required
+    directory structure.
 
     Args:
         base_dir: The base directory path (typically from CBS_BASEDIR)
@@ -95,10 +54,6 @@ def initialize_application(base_dir: str) -> None:
 
     logger.info(f"Initializing application with base directory: {base_dir}")
 
-    # Ensure directory structure
     ensure_base_structure(base_dir)
 
-    # Ensure remotes.json exists
-    ensure_remotes_json(base_dir)
-
     logger.info("Application initialization complete")

web/main.py

@@ -23,8 +23,13 @@ from web.core.logging_config import setup_logging
 from web.core.limiter import limiter, rate_limit_exceeded_handler
 
 import ap_git
-import metadata_manager
 import build_manager
+from metadata_manager import (
+    APSourceMetadataFetcher,
+    ManifestJSON,
+    VehiclesManager,
+    VersionsManager,
+)
 
 setup_logging()
 
@@ -45,20 +50,26 @@ async def lifespan(app: FastAPI):
         recurse_submodules=True,
     )
 
-    vehicles_manager = metadata_manager.VehiclesManager()
+    vehicles_manager = VehiclesManager()
+
+    manifest_json = ManifestJSON(
+        url=settings.ap_firmware_manifest_url,
+        cache_dir=settings.manifest_cache_dir,
+    )
 
-    ap_src_metadata_fetcher = metadata_manager.APSourceMetadataFetcher(
+    ap_src_metadata_fetcher = APSourceMetadataFetcher(
         ap_repo=repo,
         caching_enabled=True,
         redis_host=settings.redis_host,
         redis_port=settings.redis_port,
     )
 
-    versions_fetcher = metadata_manager.VersionsFetcher(
+    versions_manager = VersionsManager(
+        ap_repo=repo,
         remotes_json_path=settings.remotes_json_path,
-        ap_repo=repo
+        manifest_json=manifest_json,
     )
-    versions_fetcher.reload_remotes_json()
+    versions_manager.refresh_all()
 
     build_mgr = build_manager.BuildManager(
         outdir=settings.outdir_parent,
@@ -83,13 +94,14 @@ async def lifespan(app: FastAPI):
         )
         inbuilt_builder_thread.start()
 
-    versions_fetcher.start()
+    versions_manager.start()
     cleaner.start()
     progress_updater.start()
 
     app.state.repo = repo
     app.state.ap_src_metadata_fetcher = ap_src_metadata_fetcher
-    app.state.versions_fetcher = versions_fetcher
+    app.state.manifest_json = manifest_json
+    app.state.versions_manager = versions_manager
     app.state.vehicles_manager = vehicles_manager
     app.state.build_manager = build_mgr
     app.state.inbuilt_builder = inbuilt_builder
@@ -99,7 +111,7 @@ async def lifespan(app: FastAPI):
     yield
 
     # Shutdown
-    versions_fetcher.stop()
+    versions_manager.stop()
     cleaner.stop()
     progress_updater.stop()
     if inbuilt_builder is not None:

web/schemas/__init__.py

@@ -7,7 +7,7 @@ across the API endpoints.
 
 # Admin schemas
 from web.schemas.admin import (
-    RefreshRemotesResponse,
+    RefreshVersionsResponse,
 )
 
 # Build schemas
@@ -35,7 +35,7 @@ from web.schemas.vehicles import (
 
 __all__ = [
     # Admin
-    "RefreshRemotesResponse",
+    "RefreshVersionsResponse",
     # Builds
     "BuildVersionInfo",
     "RemoteInfo",

web/schemas/admin.py

@@ -3,10 +3,9 @@ from typing import List
 from pydantic import BaseModel, Field
 
 
-# --- Refresh Remotes Response ---
-class RefreshRemotesResponse(BaseModel):
-    """Response schema for remote refresh operation."""
+class RefreshVersionsResponse(BaseModel):
+    """Response schema for version metadata refresh operation."""
     remotes: List[str] = Field(
         ...,
-        description="List of remotes discovered in remotes.json file"
+        description="Git remotes synced after refreshing all version providers",
     )

web/services/admin.py

@@ -14,20 +14,20 @@ logger = logging.getLogger(__name__)
 class AdminService:
     """Service for managing administrative operations."""
 
-    def __init__(self, remote_reload_token: str, versions_fetcher=None):
+    def __init__(self, admin_token: str, versions_manager=None):
         """
         Initialize the admin service.
 
         Args:
-            remote_reload_token: Remote reload authentication token
-            versions_fetcher: VersionsFetcher instance for managing remotes
+            admin_token: Admin API authentication token
+            versions_manager: VersionsManager instance for managing remotes
         """
-        self.remote_reload_token = remote_reload_token
-        self.versions_fetcher = versions_fetcher
+        self.admin_token = admin_token
+        self.versions_manager = versions_manager
 
-    async def verify_remote_reload_token(self, token: str) -> bool:
+    async def verify_admin_token(self, token: str) -> bool:
         """
-        Verify that the provided token matches the expected remote reload token.
+        Verify that the provided token matches the expected admin token.
 
         Args:
             token: The token to verify
@@ -38,33 +38,31 @@ class AdminService:
         if not token:
             return False
 
-        return token == self.remote_reload_token
+        return token == self.admin_token
 
-    async def refresh_remotes(self) -> List[str]:
+    async def refresh_versions(self) -> List[str]:
         """
-        Trigger a refresh of remote metadata.
+        Refresh all version providers and sync git remotes.
 
         Returns:
-            List of remote names that were refreshed
+            List of git remote names synced after the refresh
 
         Raises:
             Exception: If refresh operation fails
         """
-        logger.info("Triggering remote metadata refresh")
+        logger.info("Triggering version provider refresh")
 
-        # Reload remotes.json
-        self.versions_fetcher.reload_remotes_json()
+        self.versions_manager.refresh_all()
 
-        # Get list of remotes that are now available
-        remotes_info = self.versions_fetcher.get_all_remotes_info()
-        remotes_refreshed = [remote.name for remote in remotes_info]
+        remotes_info = self.versions_manager.get_all_remotes_info()
+        remotes_synced = [remote.name for remote in remotes_info]
 
         logger.info(
-            f"Successfully refreshed {len(remotes_refreshed)} remotes: "
-            f"{remotes_refreshed}"
+            f"Successfully refreshed version providers; "
+            f"synced {len(remotes_synced)} remotes: {remotes_synced}"
         )
 
-        return remotes_refreshed
+        return remotes_synced
 
 
 def get_admin_service(
@@ -82,14 +80,14 @@ def get_admin_service(
         AdminService instance initialized with app state dependencies
 
     Raises:
-        RuntimeError: If remote reload token is not configured
+        RuntimeError: If admin token is not configured
     """
-    remote_reload_token = settings.remote_reload_token
+    admin_token = settings.admin_token
 
-    if remote_reload_token is None:
-        raise RuntimeError("Remote reload token not configured on server")
+    if admin_token is None:
+        raise RuntimeError("Admin token not configured on server")
 
     return AdminService(
-        remote_reload_token=remote_reload_token,
-        versions_fetcher=request.app.state.versions_fetcher
+        admin_token=admin_token,
+        versions_manager=request.app.state.versions_manager
     )

web/services/builds.py

@@ -29,13 +29,13 @@ class BuildsService:
     def __init__(
         self,
         build_manager=None,
-        versions_fetcher=None,
+        versions_manager=None,
         ap_src_metadata_fetcher=None,
         repo=None,
         vehicles_manager=None
     ):
         self.manager = build_manager
-        self.versions_fetcher = versions_fetcher
+        self.versions_manager = versions_manager
         self.ap_src_metadata_fetcher = ap_src_metadata_fetcher
         self.repo = repo
         self.vehicles_manager = vehicles_manager
@@ -66,7 +66,7 @@ class BuildsService:
             raise ValueError("vehicle_id is required")
 
         # Get version info using version_id
-        version_info = self.versions_fetcher.get_version_info(
+        version_info = self.versions_manager.get_version_info(
             vehicle_id=vehicle_id,
             version_id=build_request.version_id
         )
@@ -77,7 +77,7 @@ class BuildsService:
         commit_ref = version_info.commit_ref
 
         # Validate remote
-        remote_info = self.versions_fetcher.get_remote_info(remote_name)
+        remote_info = self.versions_manager.get_remote_info(remote_name)
         if remote_info is None:
             raise ValueError(f"Remote {remote_name} is not whitelisted")
 
@@ -395,7 +395,7 @@ def get_builds_service(request: Request) -> BuildsService:
     """
     return BuildsService(
         build_manager=request.app.state.build_manager,
-        versions_fetcher=request.app.state.versions_fetcher,
+        versions_manager=request.app.state.versions_manager,
         ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher,
         repo=request.app.state.repo,
         vehicles_manager=request.app.state.vehicles_manager,

web/services/vehicles.py

@@ -23,11 +23,11 @@ class VehiclesService:
     """Service for managing vehicles, versions, boards, and features."""
 
     def __init__(self, vehicle_manager=None,
-                 versions_fetcher=None,
+                 versions_manager=None,
                  ap_src_metadata_fetcher=None,
                  repo=None):
         self.vehicles_manager = vehicle_manager
-        self.versions_fetcher = versions_fetcher
+        self.versions_manager = versions_manager
         self.ap_src_metadata_fetcher = ap_src_metadata_fetcher
         self.repo = repo
 
@@ -58,7 +58,7 @@ class VehiclesService:
         """Get all versions available for a specific vehicle."""
         versions = []
 
-        for version_info in self.versions_fetcher.get_versions_for_vehicle(
+        for version_info in self.versions_manager.get_versions_for_vehicle(
             vehicle_id=vehicle_id
         ):
             # Apply type filter if provided
@@ -107,7 +107,7 @@ class VehiclesService:
     ) -> List[BoardOut]:
         """Get all boards available for a specific vehicle version."""
         # Get version info
-        version_info = self.versions_fetcher.get_version_info(
+        version_info = self.versions_manager.get_version_info(
             vehicle_id=vehicle_id,
             version_id=version_id
         )
@@ -162,7 +162,7 @@ class VehiclesService:
         vehicle version/board.
         """
         # Get version info
-        version_info = self.versions_fetcher.get_version_info(
+        version_info = self.versions_manager.get_version_info(
             vehicle_id=vehicle_id,
             version_id=version_id
         )
@@ -267,7 +267,7 @@ def get_vehicles_service(request: Request) -> VehiclesService:
     """
     return VehiclesService(
         vehicle_manager=request.app.state.vehicles_manager,
-        versions_fetcher=request.app.state.versions_fetcher,
+        versions_manager=request.app.state.versions_manager,
         ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher,
         repo=request.app.state.repo,
     )