metadata_manager: use manifest.json to get standard build urls and features.txt url

metadata_manager/__init__.py

@@ -1,5 +1,5 @@
 from .ap_src_meta_fetcher import APSourceMetadataFetcher
-from .firmware_server import ManifestJSON, ManifestFetchError, ReleaseRecord
+from .firmware_server import BoardArtifact, FeaturesTxtClient, ManifestJSON, ManifestFetchError, ReleaseRecord
 from .firmware_server.client import ManifestClient
 from .firmware_server.index import ManifestIndex
 from .vehicles_manager import DEFAULT_VEHICLES, Vehicle, VehiclesManager
@@ -18,6 +18,8 @@ from .versions_manager import (
 
 __all__ = [
     "APSourceMetadataFetcher",
+    "BoardArtifact",
+    "FeaturesTxtClient",
     "DEFAULT_VEHICLES",
     "DEFAULT_WHITELISTED_FORK_REMOTES",
     "ForkRemoteSpec",

metadata_manager/ap_src_meta_fetcher.py

@@ -478,79 +478,6 @@ class APSourceMetadataFetcher:
         )
         return build_options
 
-    def get_board_defaults_from_fw_server(
-        self,
-        artifacts_url: str,
-        board_id: str,
-        vehicle_id: str = None,
-    ) -> dict:
-        """
-        Fetch board defaults from firmware.ardupilot.org features.txt.
-
-        The features.txt file contains lines like:
-        - FEATURE_NAME (enabled features)
-        - !FEATURE_NAME (disabled features)
-
-        Parameters:
-            artifacts_url (str): Base URL for build artifacts for a version.
-            board_id (str): Board identifier
-            vehicle_id (str): Vehicle identifier
-                              (for special handling like Heli)
-
-        Returns:
-            dict: Dictionary mapping feature define to state
-                  (1 for enabled, 0 for disabled), or None if fetch fails
-        """
-        import requests
-
-        # Heli builds are stored under a separate folder
-        artifacts_subdir = board_id
-        if vehicle_id == "heli":
-            artifacts_subdir += "-heli"
-
-        features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt"
-
-        try:
-            response = requests.get(features_txt_url, timeout=30)
-            response.raise_for_status()
-
-            feature_states = {}
-            enabled_count = 0
-            disabled_count = 0
-
-            for line in response.text.splitlines():
-                line = line.strip()
-
-                # Skip empty lines and comments
-                if not line or line.startswith('#'):
-                    continue
-
-                # Check if feature is disabled (prefixed with !)
-                if line.startswith('!'):
-                    feature_name = line[1:].strip()
-                    if feature_name:
-                        feature_states[feature_name] = 0
-                        disabled_count += 1
-                else:
-                    # Enabled feature
-                    if line:
-                        feature_states[line] = 1
-                        enabled_count += 1
-
-            self.logger.info(
-                f"Fetched board defaults from firmware server: "
-                f"{enabled_count} enabled, "
-                f"{disabled_count} disabled"
-            )
-
-            return feature_states
-
-        except requests.RequestException as e:
-            self.logger.warning(
-                f"Failed to fetch board defaults from {features_txt_url}: {e}"
-            )
-            return None
-
     @staticmethod
     def get_singleton():
         return APSourceMetadataFetcher.__singleton

metadata_manager/firmware_server/__init__.py

@@ -1,8 +1,11 @@
+from .features import FeaturesTxtClient
 from .manifest import ManifestJSON
-from .models import ReleaseRecord
+from .models import BoardArtifact, ReleaseRecord
 from .exceptions import ManifestFetchError
 
 __all__ = [
+    "BoardArtifact",
+    "FeaturesTxtClient",
     "ManifestJSON",
     "ManifestFetchError",
     "ReleaseRecord",

metadata_manager/firmware_server/features.py

@@ -0,0 +1,117 @@
+import logging
+
+import dill
+import redis
+import requests
+
+FEATURES_CACHE_TTL_SEC = 86400
+
+
+def parse_features_txt(text: str) -> dict:
+    """Parse features.txt content into a define to state mapping."""
+    feature_states = {}
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line[0] == "!":
+            feature_name, state = line[1:], 0
+        else:
+            feature_name, state = line, 1
+        feature_states[feature_name] = state
+    return feature_states
+
+
+class FeaturesTxtClient:
+    """Fetch and cache board default features from firmware-server features.txt."""
+
+    def __init__(
+        self,
+        redis_host: str = "localhost",
+        redis_port: str = "6379",
+        caching_enabled: bool = True,
+        timeout: int = 30,
+        user_agent: str = "CustomBuild/1.0",
+    ):
+        self.caching_enabled = caching_enabled
+        self.timeout = timeout
+        self.user_agent = user_agent
+        self.logger = logging.getLogger(__name__)
+        self._cache_key_prefix = "features-"
+
+        if self.caching_enabled:
+            self._redis_client = redis.Redis(
+                host=redis_host,
+                port=redis_port,
+                decode_responses=False,
+            )
+
+    def _cache_key(self, url: str) -> str:
+        return self._cache_key_prefix + url
+
+    def _get_from_cache(self, url: str) -> tuple[str | None, dict | None]:
+        if not self.caching_enabled:
+            return None, None
+        value = self._redis_client.get(self._cache_key(url))
+        if value is None:
+            return None, None
+        entry = dill.loads(value)
+        if not isinstance(entry, dict):
+            return None, None
+        etag = entry.get("etag")
+        defaults = entry.get("defaults")
+        if not etag or defaults is None:
+            return None, None
+        return etag, defaults
+
+    def _store_in_cache(self, url: str, etag: str, defaults: dict) -> None:
+        if not self.caching_enabled:
+            return
+        self._redis_client.set(
+            name=self._cache_key(url),
+            value=dill.dumps({"etag": etag, "defaults": defaults}),
+            ex=FEATURES_CACHE_TTL_SEC,
+        )
+
+    def get_defaults(self, url: str) -> dict | None:
+        """
+        Fetch and parse features.txt for the given URL.
+
+        Returns a mapping of feature define to state (1 enabled, 0 disabled),
+        or None if the fetch fails.
+        """
+        cached_etag, cached_defaults = self._get_from_cache(url)
+        headers = {
+            "User-Agent": self.user_agent,
+            "Accept-Encoding": "identity",
+        }
+        if cached_etag:
+            headers["If-None-Match"] = cached_etag
+
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=self.timeout,
+            )
+            if response.status_code == 304:
+                return cached_defaults
+            response.raise_for_status()
+            defaults = parse_features_txt(response.text)
+            enabled_count = sum(1 for state in defaults.values() if state)
+            disabled_count = len(defaults) - enabled_count
+            self.logger.info(
+                "Fetched board defaults from firmware server: "
+                "%d enabled, %d disabled",
+                enabled_count,
+                disabled_count,
+            )
+            etag = response.headers.get("ETag")
+            if etag:
+                self._store_in_cache(url, etag, defaults)
+            return defaults
+        except requests.RequestException as exc:
+            self.logger.warning(
+                "Failed to fetch board defaults from %s: %s", url, exc
+            )
+            return None

metadata_manager/firmware_server/index.py

@@ -2,10 +2,43 @@ import logging
 import re
 from collections import defaultdict
 from typing import Optional
+from urllib.parse import urlparse
 
 from packaging.version import InvalidVersion, Version
 
-from .models import ReleaseRecord
+from .models import BoardArtifact, ReleaseRecord
+
+FIRMWARE_SERVER_BASE = "https://firmware.ardupilot.org"
+
+# CBS vehicle id -> firmware.ardupilot.org top-level directory.
+FIRMWARE_SERVER_DIR_BY_VEHICLE_ID = {
+    "copter": "Copter",
+    "plane": "Plane",
+    "rover": "Rover",
+    "sub": "Sub",
+    "heli": "Copter",
+    "blimp": "Blimp",
+    "tracker": "AntennaTracker",
+    "ap-periph": "AP_Periph",
+}
+
+
+def firmware_server_dir(vehicle_id: str) -> str:
+    try:
+        return FIRMWARE_SERVER_DIR_BY_VEHICLE_ID[vehicle_id]
+    except KeyError as exc:
+        raise ValueError(f"Unknown vehicle id: {vehicle_id}") from exc
+
+
+def latest_features_txt_url(vehicle_id: str, board_id: str) -> str:
+    """Hardcoded firmware-server latest features.txt URL for tag builds."""
+    board_subdir = board_id + ("-heli" if vehicle_id == "heli" else "")
+    return (
+        f"{FIRMWARE_SERVER_BASE}/"
+        f"{firmware_server_dir(vehicle_id)}/latest/"
+        f"{board_subdir}/features.txt"
+    )
+
 
 # Minimum version for a vehicle to expose from manifest entries.
 MIN_VERSION_BY_VEHICLE_ID = {
@@ -80,9 +113,52 @@ def parse_artifacts_base_url(url: str) -> Optional[str]:
     return None
 
 
+def manifest_platform_key(platform: str, vehicle_id: str) -> str:
+    """Normalize manifest platform field to the board id used for lookups."""
+    if vehicle_id == "heli" and platform.endswith("-heli"):
+        return platform[:-5]
+    return platform
+
+
+def _artifact_name_from_url(url: str) -> str:
+    return urlparse(url).path.rstrip("/").split("/")[-1]
+
+
+def _versioned_release_path_segment(release_type: str, version_number: str) -> str:
+    if version_number == "NA":
+        return release_type
+    return f"{release_type}-{version_number}"
+
+
+def _artifact_url_specificity(url: str, release_type: str, version_number: str) -> int:
+    """Prefer versioned firmware-server paths over generic release aliases."""
+    segment = _versioned_release_path_segment(release_type, version_number)
+    if f"/{segment}/" in url:
+        return 2
+    if f"/{release_type}/" in url:
+        return 1
+    return 0
+
+
+def _dedupe_board_artifacts(
+    artifacts: list[BoardArtifact],
+    release_type: str,
+    version_number: str,
+) -> list[BoardArtifact]:
+    """Manifest entries may alias the same file under versioned and generic URLs."""
+    by_name: dict[str, BoardArtifact] = {}
+    for artifact in artifacts:
+        existing = by_name.get(artifact.name)
+        if existing is None or _artifact_url_specificity(
+            artifact.url, release_type, version_number
+        ) > _artifact_url_specificity(existing.url, release_type, version_number):
+            by_name[artifact.name] = artifact
+    return list(by_name.values())
+
+
 def _release_fields_from_entry(
     entry: dict,
-) -> Optional[tuple[str, str, str, str, str]]:
+) -> Optional[tuple[str, str, str, str]]:
     vehicle_id = vehicle_id_for_manifest_entry(entry)
     if vehicle_id is None:
         return None
@@ -106,7 +182,7 @@ def _release_fields_from_entry(
         return None
 
     version_number = "NA" if release_type == "latest" else manifest_version
-    return vehicle_id, release_type, version_number, base_url, git_sha
+    return vehicle_id, release_type, version_number, git_sha
 
 
 def _record_release_meta(
@@ -114,7 +190,6 @@ def _record_release_meta(
     vehicle_id: str,
     release_type: str,
     version_number: str,
-    base_url: str,
     git_sha: str,
     logger: logging.Logger,
 ) -> None:
@@ -124,7 +199,6 @@ def _record_release_meta(
             "vehicle_id": vehicle_id,
             "release_type": release_type,
             "version_number": version_number,
-            "ap_build_artifacts_url": base_url,
             "git_sha": git_sha,
         }
         return
@@ -138,6 +212,30 @@ def _record_release_meta(
         )
 
 
+def _record_board_artifact(
+    artifacts_by_release: dict[tuple[str, str, str], dict[str, list[BoardArtifact]]],
+    entry: dict,
+    vehicle_id: str,
+    release_type: str,
+    version_number: str,
+) -> None:
+    platform = entry.get("platform")
+    artifact_url = entry.get("url", "")
+    if not platform or not artifact_url:
+        return
+
+    platform = manifest_platform_key(platform, vehicle_id)
+    key = (vehicle_id, release_type, version_number)
+    artifacts_by_release[key][platform].append(
+        BoardArtifact(
+            name=_artifact_name_from_url(artifact_url),
+            url=artifact_url,
+            format=entry.get("format", ""),
+            size=entry.get("image_size"),
+        )
+    )
+
+
 def _releases_from_meta(
     release_meta: dict[tuple, dict],
 ) -> dict[str, list[ReleaseRecord]]:
@@ -149,7 +247,6 @@ def _releases_from_meta(
                 release_type=meta["release_type"],
                 version_number=meta["version_number"],
                 commit_reference=meta["git_sha"],
-                ap_build_artifacts_url=meta["ap_build_artifacts_url"],
             )
         )
 
@@ -168,34 +265,81 @@ def _releases_from_meta(
 class ManifestIndex:
     """In-memory index built from a parsed manifest document."""
 
-    def __init__(self, releases_by_vehicle: dict[str, list[ReleaseRecord]]):
+    def __init__(
+        self,
+        releases_by_vehicle: dict[str, list[ReleaseRecord]],
+        artifacts_by_release: dict[
+            tuple[str, str, str], dict[str, list[BoardArtifact]]
+        ],
+    ):
         self.releases_by_vehicle = releases_by_vehicle
+        self.artifacts_by_release = artifacts_by_release
 
     @classmethod
     def build(cls, manifest: dict) -> "ManifestIndex":
         logger = logging.getLogger(__name__)
         release_meta: dict[tuple, dict] = {}
+        artifacts_by_release: dict[
+            tuple[str, str, str], dict[str, list[BoardArtifact]]
+        ] = defaultdict(lambda: defaultdict(list))
 
         for entry in manifest.get("firmware") or []:
             fields = _release_fields_from_entry(entry)
             if fields is None:
                 continue
-            vehicle_id, release_type, version_number, base_url, git_sha = fields
+            vehicle_id, release_type, version_number, git_sha = fields
             _record_release_meta(
                 release_meta,
                 vehicle_id,
                 release_type,
                 version_number,
-                base_url,
                 git_sha,
                 logger,
             )
+            _record_board_artifact(
+                artifacts_by_release,
+                entry,
+                vehicle_id,
+                release_type,
+                version_number,
+            )
 
-        return cls(_releases_from_meta(release_meta))
+        return cls(
+            _releases_from_meta(release_meta),
+            {
+                release_key: dict(platforms)
+                for release_key, platforms in artifacts_by_release.items()
+            },
+        )
 
     def get_releases(self, vehicle_id: str) -> list[ReleaseRecord]:
         return list(self.releases_by_vehicle.get(vehicle_id, []))
 
+    def get_board_artifacts(
+        self,
+        vehicle_id: str,
+        release_type: str,
+        version_number: str,
+        board_id: str,
+    ) -> list[BoardArtifact]:
+        key = (vehicle_id, release_type, version_number)
+        artifacts = list(self.artifacts_by_release.get(key, {}).get(board_id, []))
+        return _dedupe_board_artifacts(artifacts, release_type, version_number)
+
+    def get_features_txt_url(
+        self,
+        vehicle_id: str,
+        release_type: str,
+        version_number: str,
+        board_id: str,
+    ) -> Optional[str]:
+        artifacts = self.get_board_artifacts(
+            vehicle_id, release_type, version_number, board_id
+        )
+        if not artifacts:
+            return None
+        return artifacts[0].url.rsplit("/", 1)[0] + "/features.txt"
+
 
 def _version_sort_key(version_number: str) -> tuple:
     if version_number == "NA":

metadata_manager/firmware_server/manifest.py

@@ -3,7 +3,8 @@ from typing import Optional
 
 from .client import ManifestClient
 from .exceptions import ManifestFetchError
-from .index import ManifestIndex
+from .index import ManifestIndex, latest_features_txt_url
+from .models import BoardArtifact
 
 
 class ManifestJSON:
@@ -38,3 +39,37 @@ class ManifestJSON:
         if not self.is_available:
             return []
         return self._index.get_releases(vehicle_id)
+
+    def get_board_artifacts(
+        self,
+        vehicle_id: str,
+        release_type: str,
+        version_number: str,
+        board_id: str,
+    ) -> list[BoardArtifact]:
+        if not self.is_available:
+            return []
+        return self._index.get_board_artifacts(
+            vehicle_id=vehicle_id,
+            release_type=release_type,
+            version_number=version_number,
+            board_id=board_id,
+        )
+
+    def get_features_txt_url(
+        self,
+        vehicle_id: str,
+        release_type: str,
+        version_number: str,
+        board_id: str,
+    ) -> Optional[str]:
+        if release_type == "tag":
+            return latest_features_txt_url(vehicle_id, board_id)
+        if not self.is_available:
+            return None
+        return self._index.get_features_txt_url(
+            vehicle_id=vehicle_id,
+            release_type=release_type,
+            version_number=version_number,
+            board_id=board_id,
+        )

metadata_manager/firmware_server/models.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Optional
 
 
 @dataclass(frozen=True)
@@ -7,4 +8,11 @@ class ReleaseRecord:
     release_type: str
     version_number: str
     commit_reference: str
-    ap_build_artifacts_url: str
+
+
+@dataclass(frozen=True)
+class BoardArtifact:
+    name: str
+    url: str
+    format: str
+    size: Optional[int] = None

metadata_manager/remotes.schema.json

@@ -39,10 +39,6 @@
                       "type": "string",
                       "description": "Ardupilot version number for that release"
                     },
-                    "ap_build_artifacts_url": {
-                      "type": "string",
-                      "description": "url to build artifacts at AP firmware server to fetch features.txt"
-                    },
                     "commit_reference": {
                       "type": "string",
                       "description": "reference to commit for that release, this can be branch name, tag or git hash"

metadata_manager/versions_manager/manager.py

@@ -73,7 +73,6 @@ class VersionsManager:
             )
         self._providers = providers
         self.repo = ap_repo
-        self._remotes_json_path = remotes_json_path
         self.__task__runner = TaskRunner(tasks=((self.refresh_all, 1200),))
         VersionsManager.__singleton = self
 

metadata_manager/versions_manager/models.py

@@ -41,13 +41,11 @@ class VersionInfo:
         commit_ref: str,
         release_type: str,
         version_number: str,
-        ap_build_artifacts_url,
     ) -> None:
         self.remote_info = remote_info
         self.commit_ref = commit_ref
         self.release_type = release_type
         self.version_number = version_number
-        self.ap_build_artifacts_url = ap_build_artifacts_url
 
         commit_ref_sanitized = commit_ref.replace("/", "-")
         commit_ref_hash = hashlib.md5(commit_ref.encode()).hexdigest()[:8]
@@ -66,5 +64,4 @@ class VersionInfo:
             commit_ref=release.commit_reference,
             release_type=release.release_type,
             version_number=release.version_number,
-            ap_build_artifacts_url=release.ap_build_artifacts_url,
         )

metadata_manager/versions_manager/providers.py

@@ -18,26 +18,6 @@ from .models import ForkRemoteSpec, RemoteInfo, VersionInfo
 
 OFFICIAL_REMOTE_NAME = "ardupilot"
 OFFICIAL_REMOTE_URL = "https://github.com/ardupilot/ardupilot.git"
-FIRMWARE_SERVER_BASE = "https://firmware.ardupilot.org"
-
-# CBS vehicle id -> firmware.ardupilot.org top-level directory.
-_FIRMWARE_SERVER_DIR_BY_VEHICLE_ID = {
-    "copter": "Copter",
-    "plane": "Plane",
-    "rover": "Rover",
-    "sub": "Sub",
-    "heli": "Copter",
-    "blimp": "Blimp",
-    "tracker": "AntennaTracker",
-    "ap-periph": "AP_Periph",
-}
-
-
-def _firmware_server_dir(vehicle_id: str) -> str:
-    try:
-        return _FIRMWARE_SERVER_DIR_BY_VEHICLE_ID[vehicle_id]
-    except KeyError as exc:
-        raise ValueError(f"Unknown vehicle id: {vehicle_id}") from exc
 
 
 def _vehicle_id_for_tag_segment(segment: str) -> Optional[str]:
@@ -187,10 +167,6 @@ class WhitelistedForkTagVersionsProvider(VersionsProvider):
                             release_type="tag",
                             version_number=parts[-1],
                             commit_reference=tag_info["object"]["sha"],
-                            ap_build_artifacts_url=(
-                                f"{FIRMWARE_SERVER_BASE}/"
-                                f"{_firmware_server_dir(vid)}/latest"
-                            ),
                         )
                     )
 
@@ -294,9 +270,6 @@ class RemotesJsonVersionsProvider(VersionsProvider):
                             commit_ref=release.get("commit_reference"),
                             release_type=release.get("release_type"),
                             version_number=release.get("version_number"),
-                            ap_build_artifacts_url=release.get(
-                                "ap_build_artifacts_url"
-                            ),
                         )
                     )
         return versions