#!/usr/bin/env python3 import os import subprocess import json import pathlib import shutil import glob import time from distutils.dir_util import copy_tree from flask import Flask, render_template, request, flash from threading import Thread, Lock def get_boards(): '''return a list of boards to build''' import importlib.util spec = importlib.util.spec_from_file_location("build_binaries.py", os.path.join(basedir, "ardupilot/Tools/scripts/board_list.py")) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod.AUTOBUILD_BOARDS # list of build options to offer BUILD_OPTIONS = [ ('EKF2', 'HAL_NAVEKF2_AVAILABLE', 'Enable EKF2'), ('EKF3', 'HAL_NAVEKF3_AVAILABLE', 'Enable EKF3'), ('DSP', 'HAL_WITH_DSP', 'Enable DSP'), ('SPRAYER', 'HAL_SPRAYER_ENABLED', 'Enable Sprayer'), ('PARACHUTE', 'HAL_PARACHUTE_ENABLED', 'Enable Parachute'), ('MOUNT', 'HAL_MOUNT_ENABLED', 'Enable Mount'), ('HOTT_TELEM', 'HAL_HOTT_TELEM_ENABLED', 'Enable HoTT Telemetry'), ('BATTMON_FUEL', 'HAL_BATTMON_FUEL_ENABLE', 'Enable Fuel BatteryMonitor') ] queue_lock = Lock() from logging.config import dictConfig dictConfig({ 'version': 1, 'formatters': {'default': { 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', }}, 'handlers': {'wsgi': { 'class': 'logging.StreamHandler', 'stream': 'ext://flask.logging.wsgi_errors_stream', 'formatter': 'default' }}, 'root': { 'level': 'INFO', 'handlers': ['wsgi'] } }) #def get_template(filename): # return render_template(filename) def remove_directory_recursive(dirname): # remove a directory recursively app.logger.info('removing directory ' + dirname) if not os.path.exists(dirname): return f = pathlib.Path(dirname) if f.is_file(): f.unlink() else: shutil.rmtree(f, True) def create_directory(dir_path): # create a directory, don't fail if it exists app.logger.info('Creating ' + dir_path) pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True) def run_build(task, tmpdir, outdir): # run a build with parameters from task remove_directory_recursive(tmpdir) create_directory(tmpdir) if not os.path.isfile(os.path.join(outdir, 'extra_hwdef.dat')): app.logger.error('Build aborted, missing extra_hwdef.dat') app.logger.info('Appending to build.log') #os.remove(os.path.join(outdir, 'build.log')) with open(os.path.join(outdir, 'build.log'), 'a') as log: app.logger.info('Submodule update') subprocess.run(['git', 'submodule', 'update', '--recursive', '--force', '--init'], stdout=log, stderr=log) app.logger.info('Running waf configure') subprocess.run(['./waf', 'configure', '--board', task['board'], '--out', tmpdir, '--extra-hwdef', task['extra_hwdef']], cwd = task['sourcedir'], stdout=log, stderr=log) app.logger.info('Running clean') subprocess.run(['./waf', 'clean'], cwd = task['sourcedir'], stdout=log, stderr=log) app.logger.info('Running build') subprocess.run(['./waf', task['vehicle']], cwd = task['sourcedir'], stdout=log, stderr=log) # background thread to check for queued build requests def check_queue(): queue_lock.acquire() json_files = list(filter(os.path.isfile, glob.glob(os.path.join(outdir_parent, '*', 'q.json')))) json_files.sort(key=lambda x: os.path.getmtime(x)) queue_lock.release() if len(json_files) == 0: return taskfile = json_files[0] app.logger.info('Opening ' + taskfile) task = json.loads(open(taskfile).read()) os.remove(taskfile) outdir = os.path.join(outdir_parent, task['token']) tmpdir = os.path.join(tmpdir_parent, task['token']) try: # run build and rename build directory run_build(task, tmpdir, outdir) app.logger.info('Copying build files from %s to %s', os.path.join(tmpdir, task['board']), outdir) copy_tree(os.path.join(tmpdir, task['board'], 'bin'), outdir) app.logger.info('Build successful!') app.logger.info('Removing ' + tmpdir) remove_directory_recursive(tmpdir) # remove extra_hwdef.dat app.logger.info('Removing ' + os.path.join(outdir, 'extra_hwdef.dat')) os.remove(os.path.join(outdir, 'extra_hwdef.dat')) except Exception as ex: app.logger.info('Build failed') print(ex) pass def queue_thread(): while True: try: check_queue() time.sleep(5) except Exception as ex: print(ex) print('Failed queue: ', ex) pass import optparse parser = optparse.OptionParser("app.py") parser.add_option("", "--basedir", type="string", default="..", help="base directory") cmd_opts, cmd_args = parser.parse_args() # define directories basedir = os.path.abspath(cmd_opts.basedir) sourcedir = os.path.abspath(os.path.join(basedir, 'ardupilot')) outdir_parent = os.path.join(basedir, 'builds') tmpdir_parent = os.path.join(basedir, 'tmp') # Directory of this file this_path = os.path.dirname(os.path.realpath(__file__)) # Where the user requested tile are stored output_path = os.path.join(this_path, '..', 'userRequestFirmware') # Where the data database is tile_path = os.path.join(this_path, '..', 'data', 'tiles') # The output folder for all gzipped build requests app = Flask(__name__, static_url_path='/builds', static_folder=output_path, template_folder='templates') if not os.path.isdir(outdir_parent): create_directory(outdir_parent) thread = Thread(target=queue_thread, args=()) thread.daemon = True thread.start() @app.route('/generate', methods=['GET', 'POST']) def generate(): #if request.method == 'POST': features = [] task = {} try: # fetch features from user input app.logger.info('Fetching features from user input') for (label, define, text) in BUILD_OPTIONS: if label not in request.form: continue value = request.form[label] features.append(value) undefine = 'undef ' + define features.insert(0,undefine) extra_hwdef = '\n'.join(features) queue_lock.acquire() # create extra_hwdef.dat file and obtain md5sum app.logger.info('Creating ' + os.path.join(outdir_parent, 'extra_hwdef.dat')) file = open(os.path.join(outdir_parent, 'extra_hwdef.dat'),'w') app.logger.info('Writing\n' + extra_hwdef) file.write(extra_hwdef) file.close() app.logger.info('Getting md5sum') md5sum = subprocess.check_output(['md5sum', os.path.join(outdir_parent, 'extra_hwdef.dat')], encoding = 'utf-8') md5sum = md5sum[:len(md5sum) -(3+len(os.path.join(outdir_parent, 'extra_hwdef.dat')))] app.logger.info('md5sum = ' + md5sum) app.logger.info('Removing ' + os.path.join(outdir_parent, 'extra_hwdef.dat')) os.remove(os.path.join(outdir_parent, 'extra_hwdef.dat')) # obtain git-hash of source app.logger.info('Getting git hash') git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd = sourcedir, encoding = 'utf-8') git_hash = git_hash[:len(git_hash)-1] app.logger.info('Git hash = ' + git_hash) # create directories using concatenated token # of vehicle, board, git-hash of source, and md5sum of hwdef vehicle = request.form['vehicle'] board = request.form['board'] token = vehicle + '-' + board + '-' + git_hash + '-' + md5sum app.logger.info('token = ' + token) outdir = os.path.join(outdir_parent, token) if os.path.isdir(outdir): app.logger.info('Build already exists') else: app.logger.info('Creating ' + outdir) create_directory(outdir) # create build.log app.logger.info('Creating build.log') build_log = open(os.path.join(outdir, 'build.log'), 'w') build_log.write('Waiting for build to start...\n') build_log.close() # create hwdef.dat app.logger.info('Opening ' + os.path.join(outdir, 'extra_hwdef.dat')) file = open(os.path.join(outdir, 'extra_hwdef.dat'),'w') app.logger.info('Writing\n' + extra_hwdef) file.write(extra_hwdef) file.close() # fill dictionary of variables and create json file task['hwdef_md5sum'] = md5sum task['git_hash'] = git_hash task['token'] = token task['sourcedir'] = sourcedir task['extra_hwdef'] = os.path.join(outdir, 'extra_hwdef.dat') task['board'] = board task['vehicle'] = vehicle app.logger.info('Opening ' + os.path.join(outdir, 'q.json')) jfile = open(os.path.join(outdir, 'q.json'), 'w') app.logger.info('Writing task file to ' + os.path.join(outdir, 'q.json')) jfile.write(json.dumps(task)) jfile.close() queue_lock.release() apache_build_dir = 'http://localhost:8080/' \ + os.path.join('builds', token) apache_build_log = 'http://localhost:8080/' \ + os.path.join('builds', token, 'build.log') apache_all_builds = 'http://localhost:8080/' \ + 'builds' app.logger.info('Rendering generate.html') return render_template('generate.html', apache_build_dir=apache_build_dir, apache_build_log=apache_build_log, apache_all_builds=apache_all_builds, token=token) except Exception as ex: print(ex) return render_template('generate.html', error='Error occured') def get_build_options(): return BUILD_OPTIONS @app.route('/') @app.route('/home', methods=['POST']) def home(): app.logger.info('Rendering index.html') return render_template('index.html', get_boards=get_boards, get_build_options=get_build_options) if __name__ == '__main__': app.run()