ArdupilotCustomFirmwareBuilder/app.py

#!/usr/bin/env python3

import uuid
import os
import sys
import subprocess
import zipfile
import urllib.request
import gzip
from io import BytesIO
import time
import json
import pathlib
import shutil
from distutils.dir_util import copy_tree
import logging

from flask import Flask, render_template, request, flash
from threading import Thread, Lock

queue_lock = Lock()

from logging.config import dictConfig

dictConfig({
    'version': 1,
    'formatters': {'default': {
        'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
    }},
    'handlers': {'wsgi': {
        'class': 'logging.StreamHandler',
        'stream': 'ext://flask.logging.wsgi_errors_stream',
        'formatter': 'default'
    }},
    'root': {
        'level': 'INFO',
        'handlers': ['wsgi']
    }
})

'''
Directory structure:
  - all paths relative to where the app starts
  - waf builds go into build
  - json queued build files go into buildqueue
  - resulting builds go into done
  - ardupilot is in ardupilot directory
  - templates in CustomBuild/templates

'''

#def get_template(filename):
#    return render_template(filename)

def remove_directory_recursive(dirname):
    # remove a directory recursively
    app.logger.info('removing directory ' + dirname)
    if not os.path.exists(dirname):
        return
    f = pathlib.Path(dirname)
    if f.is_file():
        f.unlink()
    else:
        shutil.rmtree(f, True)


def create_directory(dir_path):
    # create a directory, don't fail if it exists
    app.logger.info('creating ' + dir_path)
    pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)


def run_build(taskfile, builddir, done_dir):
    # run a build with parameters from task
    app.logger.info('Opening q.json')
    task = json.loads(open(taskfile).read())
    remove_directory_recursive(os.path.join(sourcedir, done_dir))
    remove_directory_recursive(os.path.join(sourcedir, builddir))
    create_directory(os.path.join(sourcedir, done_dir))
    create_directory(os.path.join(sourcedir, builddir))
    app.logger.info('Creating build.log')
    with open(os.path.join(sourcedir, done_dir, 'build.log'), "wb") as log:
        app.logger.info('Submodule update')
        subprocess.run(['git', 'submodule',
                        'update', '--recursive', 
                        '--force', '--init'], stdout=log, stderr=log)
        app.logger.info('Running waf configure')
        subprocess.run(['./waf', 'configure', 
                        '--board', task['board'], 
                        '--out', builddir, 
                        '--extra-hwdef', task['extra_hwdef']],
                        cwd = task['sourcedir'], 
                        stdout=log, stderr=log)
        app.logger.info('Running clean')
        subprocess.run(['./waf', 'clean'], cwd = task['sourcedir'], 
                        stdout=log, stderr=log)
        app.logger.info('Running build')
        subprocess.run(['./waf', task['vehicle']], cwd = task['sourcedir'], 
                        stdout=log, stderr=log)

# background thread to check for queued build requests
def check_queue():
    while(1):
        queue_lock.acquire()
        listing = os.listdir('buildqueue')
        queue_lock.release()
        if listing:
            for token in listing:
                builddir = 'build'
                done_dir = os.path.join('done', token)
                buildqueue_dir = os.path.join('buildqueue', token)
                # check if build exists
                if os.path.isdir(os.path.join(sourcedir, done_dir)):
                    app.logger.info('Build already exists')
                else:
                    try:
                        # run build and rename build directory
                        app.logger.info('Creating ' + 
                                        os.path.join(buildqueue_dir, 'q.json'))
                        f = open(os.path.join(buildqueue_dir, 'q.json'))
                        app.logger.info('Loading ' + 
                                        os.path.join(buildqueue_dir, 'q.json'))
                        task = json.load(f)
                        run_build(os.path.join(buildqueue_dir, 'q.json'), 
                                    builddir, done_dir)
                        app.logger.info('Copying build files from %s to %s', 
                                        os.path.join(sourcedir, builddir, task['board']),
                                        os.path.join(sourcedir, done_dir))
                        copy_tree(os.path.join(sourcedir, builddir, task['board']),
                                    os.path.join(sourcedir, done_dir))
                        app.logger.info('Build successful!')
                    
                    except:
                        app.logger.info('Build failed')
                        continue
                
                # remove working files
                app.logger.info('Removing ' + 
                                os.path.join(buildqueue_dir, 'extra_hwdef.dat'))
                os.remove(os.path.join(buildqueue_dir, 'extra_hwdef.dat'))
                app.logger.info('Removing ' + 
                                os.path.join(buildqueue_dir, 'q.json'))
                os.remove(os.path.join(buildqueue_dir, 'q.json'))
                app.logger.info('Removing ' + 
                                os.path.join(buildqueue_dir))
                os.rmdir(os.path.join(buildqueue_dir))

# define source and app directories
sourcedir = os.path.abspath(os.path.join('..', 'ardupilot'))
appdir = os.path.abspath(os.curdir)

if not os.path.isdir('buildqueue'):
    os.mkdir('buildqueue')

thread = Thread(target=check_queue, args=())
thread.daemon = True
thread.start()

# Directory of this file
this_path = os.path.dirname(os.path.realpath(__file__))

# Where the user requested tile are stored
output_path = os.path.join(this_path, '..', 'userRequestFirmware')

# Where the data database is
tile_path = os.path.join(this_path, '..', 'data', 'tiles')

# The output folder for all gzipped build requests
app = Flask(__name__, static_url_path='/builds', 
            static_folder=output_path, template_folder='templates')

@app.route('/')
def index():
    app.logger.info('Rendering index.html')
    return render_template('index.html')

@app.route('/generate', methods=['GET', 'POST'])
def generate():
    #if request.method == 'POST':
    features = []
    task = {}

    try:
        # fetch features from user input
        app.logger.info('Fetching features from user input')
        for i in range(1,8):
            value = request.form["option" + str(i)]
            features.append(value)
            undefine = "undef " + value.split()[1]
            features.insert(0,undefine)
        extra_hwdef = '\n'.join(features)

        #print("features: ", features)

        queue_lock.acquire()

        # create extra_hwdef.dat file and obtain md5sum
        app.logger.info('Creating ' + os.path.join('buildqueue', 'extra_hwdef.dat'))
        file = open(os.path.join('buildqueue', 'extra_hwdef.dat'),"w")
        app.logger.info('Writing\n' + extra_hwdef)
        file.write(extra_hwdef)
        file.close()
        app.logger.info('Getting md5sum')
        md5sum = subprocess.check_output(['md5sum', 'buildqueue/extra_hwdef.dat'],
                                            encoding = 'utf-8')
        md5sum = md5sum[:len(md5sum)-29]
        app.logger.info('md5sum = ' + md5sum)
        app.logger.info('Removing ' + os.path.join('buildqueue', 'extra_hwdef.dat'))
        os.remove(os.path.join('buildqueue', 'extra_hwdef.dat'))

        # obtain git-hash of source
        app.logger.info('Getting git hash')
        git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], 
                                            cwd = sourcedir,
                                            encoding = 'utf-8')
        git_hash = git_hash[:len(git_hash)-1]
        app.logger.info('Git hash = ' + git_hash)

        # create directories using concatenated token of git-hash and md5sum of hwdef
        token = git_hash + "-" + md5sum
        app.logger.info('token = ' + token)
        buildqueue_dir = os.path.join(appdir, 'buildqueue', token)
        if not os.path.isdir(buildqueue_dir):
            app.logger.info('Creating ' + buildqueue_dir)
            os.mkdir(buildqueue_dir)
        app.logger.info('Opening ' + os.path.join(buildqueue_dir, 'extra_hwdef.dat'))
        file = open(os.path.join(buildqueue_dir, 'extra_hwdef.dat'),"w")
        app.logger.info('Writing\n' + extra_hwdef)
        file.write(extra_hwdef)
        file.close()
        
        # fill dictionary of variables and create json file
        task['hwdef_md5sum'] = md5sum
        task['git_hash'] = git_hash
        task['token'] = token
        task['sourcedir'] = sourcedir
        task['extra_hwdef'] = os.path.join(buildqueue_dir, 'extra_hwdef.dat')
        task['board'] = request.form["board"]
        task['vehicle'] = request.form["vehicle"]
        app.logger.info('Opening ' + os.path.join(buildqueue_dir, 'q.json'))
        jfile = open(os.path.join(buildqueue_dir, 'q.json'), "w")
        app.logger.info('Writing task file to ' + os.path.join(buildqueue_dir, 'q.json'))
        jfile.write(json.dumps(task))
        jfile.close()

        queue_lock.release()

        #print(task)

        apache_build_dir = "http://localhost:8080/" + token
        apache_build_log = "http://localhost:8080/" + token + "/build.log"
        app.logger.info('Rendering generate.html')
        return render_template('generate.html', 
                                apache_build_dir=apache_build_dir, 
                                apache_build_log=apache_build_log,
                                token=token)
    
    except:
        return render_template('generate.html', error='Error occured')

@app.route('/home', methods=['POST'])
def home():
    app.logger.info('Rendering index.html')
    return render_template('index.html')

if __name__ == "__main__":
    app.run()