ArdupilotCustomFirmwareBuilder/web/app.py

#!/usr/bin/env python3

import os
import base64
from flask import Flask, render_template, request, send_from_directory, jsonify, redirect
from threading import Thread
import sys
import requests

from logging.config import dictConfig

dictConfig({
    'version': 1,
    'formatters': {'default': {
        'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
    }},
    'handlers': {'wsgi': {
        'class': 'logging.StreamHandler',
        'stream': 'ext://flask.logging.wsgi_errors_stream',
        'formatter': 'default'
    }},
    'root': {
        'level': os.getenv('CBS_LOG_LEVEL', default='INFO'),
        'handlers': ['wsgi']
    }
})

# let app.py know about the modules in the parent directory
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
import ap_git
import metadata_manager
import build_manager
from builder import Builder

# run at lower priority
os.nice(20)

import optparse
parser = optparse.OptionParser("app.py")

parser.add_option("", "--basedir", type="string",
                  default=os.getenv(
                      key="CBS_BASEDIR",
                      default=os.path.abspath(os.path.join(os.path.dirname(__file__),"..","base"))
                  ),
                  help="base directory")

cmd_opts, cmd_args = parser.parse_args()

# define directories
basedir = os.path.abspath(cmd_opts.basedir)
sourcedir = os.path.join(basedir, 'ardupilot')
outdir_parent = os.path.join(basedir, 'builds')
workdir_parent = os.path.join(basedir, 'workdir')

appdir = os.path.dirname(__file__)

builds_dict = {}
REMOTES = None

repo = ap_git.GitRepo.clone_if_needed(
    source="https://github.com/ardupilot/ardupilot.git",
    dest=sourcedir,
    recurse_submodules=True,
)

ap_src_metadata_fetcher = metadata_manager.APSourceMetadataFetcher(
    ap_repo=repo,
    caching_enabled=True,
    redis_host=os.getenv('CBS_REDIS_HOST', default='localhost'),
    redis_port=os.getenv('CBS_REDIS_PORT', default='6379'),
)
versions_fetcher = metadata_manager.VersionsFetcher(
    remotes_json_path=os.path.join(basedir, 'configs', 'remotes.json'),
    ap_repo=repo
)

manager = build_manager.BuildManager(
    outdir=outdir_parent,
    redis_host=os.getenv('CBS_REDIS_HOST', default='localhost'),
    redis_port=os.getenv('CBS_REDIS_PORT', default='6379')
)
cleaner = build_manager.BuildArtifactsCleaner()
progress_updater = build_manager.BuildProgressUpdater()

versions_fetcher.start()
cleaner.start()
progress_updater.start()

if os.getenv('CBS_ENABLE_INBUILT_BUILDER', default='1') == '1':
    builder = Builder(
        workdir=workdir_parent,
        source_repo=repo
    )
    builder_thread = Thread(
        target=builder.run,
        daemon=True
    )
    builder_thread.start()

app = Flask(__name__, template_folder='templates')

versions_fetcher.reload_remotes_json()
app.logger.info('Python version is: %s' % sys.version)

def get_auth_token():
    try:
        # try to read the secret token from the file
        with open(os.path.join(basedir, 'secrets', 'reload_token'), 'r') as file:
            token = file.read().strip()
            return token
    except (FileNotFoundError, PermissionError):
        app.logger.error("Couldn't open token file. Checking environment for token.")
        # if the file does not exist, check the environment variable
        return os.getenv('CBS_REMOTES_RELOAD_TOKEN')

@app.route('/refresh_remotes', methods=['POST'])
def refresh_remotes():
    auth_token = get_auth_token()

    if auth_token is None:
        app.logger.error("Couldn't retrieve authorization token")
        return "Internal Server Error", 500

    token = request.get_json().get('token')
    if not token or token != auth_token:
        return "Unauthorized", 401

    versions_fetcher.reload_remotes_json()
    return "Successfully refreshed remotes", 200

@app.route('/generate', methods=['GET', 'POST'])
def generate():
    try:
        version = request.form['version']
        remote_name, commit_ref = version.split('/', 1)
        remote_info = versions_fetcher.get_remote_info(remote_name)

        if remote_info is None:
            raise Exception(f"Remote {remote_name} is not whitelisted.")

        vehicle = request.form['vehicle']
        version_info = versions_fetcher.get_version_info(
            vehicle=vehicle,
            remote=remote_name,
            commit_ref=commit_ref
        )

        if version_info is None:
            raise Exception("Commit reference invalid or not listed to be built for given vehicle for remote")

        board = request.form['board']
        boards_at_commit = ap_src_metadata_fetcher.get_boards_at_commit(
            remote=remote_name,
            commit_ref=commit_ref
        )
        if board not in boards_at_commit:
            raise Exception("bad board")

        all_features = ap_src_metadata_fetcher.get_build_options_at_commit(
            remote=remote_name,
            commit_ref=commit_ref
        )

        chosen_defines = {
            feature.define
            for feature in all_features
            if request.form.get(feature.label) == "1"
        }

        git_hash = repo.commit_id_for_remote_ref(
            remote=remote_name,
            commit_ref=commit_ref
        )

        build_info = build_manager.BuildInfo(
            vehicle=vehicle,
            remote_info=remote_info,
            git_hash=git_hash,
            board=board,
            selected_features=chosen_defines
        )

        forwarded_for = request.headers.get('X-Forwarded-For', None)
        if forwarded_for:
            client_ip = forwarded_for.split(',')[0].strip()
        else:
            client_ip = request.remote_addr

        build_id = manager.submit_build(
            build_info=build_info,
            client_ip=client_ip,
        )

        app.logger.info('Redirecting to /viewlog')
        return redirect('/viewlog/'+build_id)

    except Exception as ex:
        app.logger.error(ex)
        return render_template('error.html', ex=ex)

@app.route('/add_build')
def add_build():
    app.logger.info('Rendering add_build.html')
    return render_template('add_build.html')


def filter_build_options_by_category(build_options, category):
    return sorted([f for f in build_options if f.category == category], key=lambda x: x.description.lower())

def parse_build_categories(build_options):
    return sorted(list(set([f.category for f in build_options])))

@app.route('/', defaults={'token': None}, methods=['GET'])
@app.route('/viewlog/<token>', methods=['GET'])
def home(token):
    if token:
        app.logger.info("Showing log for build id " + token)
    app.logger.info('Rendering index.html')
    return render_template('index.html', token=token)

@app.route("/builds/<path:name>")
def download_file(name):
    app.logger.info('Downloading %s' % name)
    return send_from_directory(os.path.join(basedir,'builds'), name, as_attachment=False)

@app.route("/boards_and_features/<string:vehicle_name>/<string:remote_name>/<string:commit_reference>", methods=['GET'])
def boards_and_features(vehicle_name, remote_name, commit_reference):
    commit_reference = base64.urlsafe_b64decode(commit_reference).decode()

    if not versions_fetcher.is_version_listed(vehicle=vehicle_name, remote=remote_name, commit_ref=commit_reference):
        return "Bad request. Commit reference not allowed to build for the vehicle.", 400

    app.logger.info('Board list and build options requested for %s %s %s' % (vehicle_name, remote_name, commit_reference))
    # getting board list for the branch
    with repo.get_checkout_lock():
        boards = ap_src_metadata_fetcher.get_boards_at_commit(
            remote=remote_name,
            commit_ref=commit_reference
        )

        options = ap_src_metadata_fetcher.get_build_options_at_commit(
            remote=remote_name,
            commit_ref=commit_reference
        )   # this is a list of Feature() objects defined in build_options.py

    # parse the set of categories from these objects
    categories = parse_build_categories(options)
    features = []
    for category in categories:
        filtered_options = filter_build_options_by_category(options, category)
        category_options = []   # options belonging to a given category
        for option in filtered_options:
            category_options.append({
                'label' : option.label,
                'description' : option.description,
                'default' : option.default,
                'define' : option.define,
                'dependency' : option.dependency,
            })
        features.append({
            'name' : category,
            'options' : category_options,
        })
    # creating result dictionary
    result = {
        'boards' : boards,
        'default_board' : boards[0],
        'features' : features,
    }
    # return jsonified result dict
    return jsonify(result)

@app.route("/get_versions/<string:vehicle_name>", methods=['GET'])
def get_versions(vehicle_name):
    versions = list()
    for version_info in versions_fetcher.get_versions_for_vehicle(vehicle_name=vehicle_name):
        if version_info.release_type == "latest":
            title = f"Latest ({version_info.remote})"
        else:
            title = f"{version_info.release_type} {version_info.version_number} ({version_info.remote})"
        id = f"{version_info.remote}/{version_info.commit_ref}"
        versions.append({
            "title" :   title,
            "id"    :   id,
        })

    return jsonify(sorted(versions, key=lambda x: x['title']))

@app.route("/get_vehicles")
def get_vehicles():
    return jsonify(versions_fetcher.get_all_vehicles_sorted_uniq())

@app.route("/get_defaults/<string:vehicle_name>/<string:remote_name>/<string:commit_reference>/<string:board_name>", methods = ['GET'])
def get_deafults(vehicle_name, remote_name, commit_reference, board_name):
    # Heli is built on copter
    if vehicle_name == "Heli":
        vehicle_name = "Copter"

    commit_reference = base64.urlsafe_b64decode(commit_reference).decode()
    version_info = versions_fetcher.get_version_info(vehicle=vehicle_name, remote=remote_name, commit_ref=commit_reference)

    if version_info is None:
        return "Bad request. Commit reference %s is not allowed for builds for the %s for %s remote." % (commit_reference, vehicle_name, remote_name), 400

    artifacts_dir = version_info.ap_build_artifacts_url

    if artifacts_dir is None:
        return "Couldn't find artifacts for requested release/branch/commit on ardupilot server", 404

    url_to_features_txt = artifacts_dir + '/' + board_name + '/features.txt'
    response = requests.get(url_to_features_txt, timeout=30)

    if not response.status_code == 200:
        return ("Could not retrieve features.txt for given vehicle, version and board combination (Status Code: %d, url: %s)" % (response.status_code, url_to_features_txt), response.status_code)
    # split response by new line character to get a list of defines
    result = response.text.split('\n')
    # omit the last two elements as they are always blank
    return jsonify(result[:-2])

if __name__ == '__main__':
    app.run()