updated directory structure and fixed multiple build request issues

app.py

@@ -1,20 +1,12 @@
 #!/usr/bin/env python3
 
-import uuid
 import os
-import sys
 import subprocess
-import zipfile
-import urllib.request
-import gzip
-from io import BytesIO
-import time
 import json
 import pathlib
 import shutil
+import glob
 from distutils.dir_util import copy_tree
-import logging
-
 from flask import Flask, render_template, request, flash
 from threading import Thread, Lock
 
@@ -38,17 +30,6 @@ dictConfig({
     }
 })
 
-'''
-Directory structure:
-  - all paths relative to where the app starts
-  - waf builds go into build
-  - json queued build files go into buildqueue
-  - resulting builds go into done
-  - ardupilot is in ardupilot directory
-  - templates in CustomBuild/templates
-
-'''
-
 #def get_template(filename):
 #    return render_template(filename)
 
@@ -66,28 +47,32 @@ def remove_directory_recursive(dirname):
 
 def create_directory(dir_path):
     # create a directory, don't fail if it exists
-    app.logger.info('creating ' + dir_path)
+    app.logger.info('Creating ' + dir_path)
     pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
 
 
-def run_build(taskfile, builddir, done_dir):
+def run_build(taskfile, tmpdir, outdir):
     # run a build with parameters from task
-    app.logger.info('Opening q.json')
+    app.logger.info('Opening ' + taskfile)
     task = json.loads(open(taskfile).read())
-    remove_directory_recursive(os.path.join(sourcedir, done_dir))
-    remove_directory_recursive(os.path.join(sourcedir, builddir))
-    create_directory(os.path.join(sourcedir, done_dir))
-    create_directory(os.path.join(sourcedir, builddir))
+    app.logger.info('Removing ' + taskfile)
+    os.remove(taskfile)
+    remove_directory_recursive(tmpdir)
+    create_directory(tmpdir)
+    if not os.path.isfile(os.path.join(outdir, 'extra_hwdef.dat')):
+        app.logger.error('Build aborted, missing extra_hwdef.dat')
     app.logger.info('Creating build.log')
-    with open(os.path.join(sourcedir, done_dir, 'build.log'), "wb") as log:
+    #os.remove(os.path.join(outdir, 'build.log'))
+    with open(os.path.join(outdir, 'build.log'), 'a') as log:
         app.logger.info('Submodule update')
         subprocess.run(['git', 'submodule',
                         'update', '--recursive', 
-                        '--force', '--init'], stdout=log, stderr=log)
+                        '--force', '--init'], 
+                        stdout=log, stderr=log)
         app.logger.info('Running waf configure')
         subprocess.run(['./waf', 'configure', 
                         '--board', task['board'], 
-                        '--out', builddir, 
+                        '--out', tmpdir, 
                         '--extra-hwdef', task['extra_hwdef']],
                         cwd = task['sourcedir'], 
                         stdout=log, stderr=log)
@@ -102,59 +87,53 @@ def run_build(taskfile, builddir, done_dir):
 def check_queue():
     while(1):
         queue_lock.acquire()
-        listing = os.listdir('buildqueue')
+        json_files = list(filter(os.path.isfile, 
+                                    glob.glob(os.path.join(outdir_parent, 
+                                                '*', 'q.json'))))
+        json_files.sort(key=lambda x: os.path.getmtime(x))
         queue_lock.release()
-        if listing:
-            for token in listing:
-                builddir = 'build'
-                done_dir = os.path.join('done', token)
-                buildqueue_dir = os.path.join('buildqueue', token)
+        if json_files:
+            for taskfile in json_files:
+                #taskfile = os.path.join(outdir, file)
+                app.logger.info('Opening ' + taskfile)
+                task = json.loads(open(taskfile).read())
+                outdir = os.path.join(outdir_parent, task['token'])
+                tmpdir = os.path.join(tmpdir_parent, task['token'])
                 # check if build exists
-                if os.path.isdir(os.path.join(sourcedir, done_dir)):
-                    app.logger.info('Build already exists')
-                else:
-                    try:
-                        # run build and rename build directory
-                        app.logger.info('Creating ' + 
-                                        os.path.join(buildqueue_dir, 'q.json'))
-                        f = open(os.path.join(buildqueue_dir, 'q.json'))
-                        app.logger.info('Loading ' + 
-                                        os.path.join(buildqueue_dir, 'q.json'))
-                        task = json.load(f)
-                        run_build(os.path.join(buildqueue_dir, 'q.json'), 
-                                    builddir, done_dir)
-                        app.logger.info('Copying build files from %s to %s', 
-                                        os.path.join(sourcedir, builddir, task['board']),
-                                        os.path.join(sourcedir, done_dir))
-                        copy_tree(os.path.join(sourcedir, builddir, task['board']),
-                                    os.path.join(sourcedir, done_dir))
-                        app.logger.info('Build successful!')
-                    
-                    except:
-                        app.logger.info('Build failed')
-                        continue
+                #if os.path.isdir(outdir):
+                    #app.logger.info('Build already exists')
+                    #app.logger.info('Removing ' + taskfile)
+                    #os.remove(taskfile)
+                #else:
+                try:
+                    # run build and rename build directory
+                    app.logger.info('Opening ' + taskfile)
+                    f = open(taskfile)
+                    app.logger.info('Loading ' + taskfile)
+                    task = json.load(f)
+                    run_build(taskfile, tmpdir, outdir)
+                    app.logger.info('Copying build files from %s to %s', 
+                                    os.path.join(tmpdir, task['board']),
+                                    outdir)
+                    copy_tree(os.path.join(tmpdir, task['board'], 'bin'), 
+                                outdir)
+                    app.logger.info('Build successful!')
+                    app.logger.info('Removing ' + tmpdir)
+                    remove_directory_recursive(tmpdir)
+                    # remove extra_hwdef.dat
+                    app.logger.info('Removing ' + 
+                                    os.path.join(outdir, 'extra_hwdef.dat'))
+                    os.remove(os.path.join(outdir, 'extra_hwdef.dat'))
                 
-                # remove working files
-                app.logger.info('Removing ' + 
-                                os.path.join(buildqueue_dir, 'extra_hwdef.dat'))
-                os.remove(os.path.join(buildqueue_dir, 'extra_hwdef.dat'))
-                app.logger.info('Removing ' + 
-                                os.path.join(buildqueue_dir, 'q.json'))
-                os.remove(os.path.join(buildqueue_dir, 'q.json'))
-                app.logger.info('Removing ' + 
-                                os.path.join(buildqueue_dir))
-                os.rmdir(os.path.join(buildqueue_dir))
-
-# define source and app directories
-sourcedir = os.path.abspath(os.path.join('..', 'ardupilot'))
-appdir = os.path.abspath(os.curdir)
+                except:
+                    app.logger.info('Build failed')
+                    continue
 
-if not os.path.isdir('buildqueue'):
-    os.mkdir('buildqueue')
-
-thread = Thread(target=check_queue, args=())
-thread.daemon = True
-thread.start()
+# define directories
+basedir = os.path.abspath('..')
+sourcedir = os.path.abspath(os.path.join(basedir, 'ardupilot'))
+outdir_parent = os.path.join(basedir, 'builds')
+tmpdir_parent = os.path.join(basedir, 'tmp')
 
 # Directory of this file
 this_path = os.path.dirname(os.path.realpath(__file__))
@@ -169,6 +148,13 @@ tile_path = os.path.join(this_path, '..', 'data', 'tiles')
 app = Flask(__name__, static_url_path='/builds', 
             static_folder=output_path, template_folder='templates')
 
+if not os.path.isdir(outdir_parent):
+    create_directory(outdir_parent)
+
+thread = Thread(target=check_queue, args=())
+thread.daemon = True
+thread.start()
+
 @app.route('/')
 def index():
     app.logger.info('Rendering index.html')
@@ -184,29 +170,32 @@ def generate():
         # fetch features from user input
         app.logger.info('Fetching features from user input')
         for i in range(1,8):
-            value = request.form["option" + str(i)]
+            value = request.form['option' + str(i)]
             features.append(value)
-            undefine = "undef " + value.split()[1]
+            undefine = 'undef ' + value.split()[1]
             features.insert(0,undefine)
         extra_hwdef = '\n'.join(features)
 
-        #print("features: ", features)
-
         queue_lock.acquire()
 
         # create extra_hwdef.dat file and obtain md5sum
-        app.logger.info('Creating ' + os.path.join('buildqueue', 'extra_hwdef.dat'))
-        file = open(os.path.join('buildqueue', 'extra_hwdef.dat'),"w")
+        app.logger.info('Creating ' + 
+                        os.path.join(outdir_parent, 'extra_hwdef.dat'))
+        file = open(os.path.join(outdir_parent, 'extra_hwdef.dat'),'w')
         app.logger.info('Writing\n' + extra_hwdef)
         file.write(extra_hwdef)
         file.close()
         app.logger.info('Getting md5sum')
-        md5sum = subprocess.check_output(['md5sum', 'buildqueue/extra_hwdef.dat'],
+        md5sum = subprocess.check_output(['md5sum', 
+                                            os.path.join(outdir_parent, 
+                                            'extra_hwdef.dat')],
                                             encoding = 'utf-8')
-        md5sum = md5sum[:len(md5sum)-29]
+        md5sum = md5sum[:len(md5sum)
+                        -(3+len(os.path.join(outdir_parent, 'extra_hwdef.dat')))]
         app.logger.info('md5sum = ' + md5sum)
-        app.logger.info('Removing ' + os.path.join('buildqueue', 'extra_hwdef.dat'))
-        os.remove(os.path.join('buildqueue', 'extra_hwdef.dat'))
+        app.logger.info('Removing ' + 
+                        os.path.join(outdir_parent, 'extra_hwdef.dat'))
+        os.remove(os.path.join(outdir_parent, 'extra_hwdef.dat'))
 
         # obtain git-hash of source
         app.logger.info('Getting git hash')
@@ -216,43 +205,57 @@ def generate():
         git_hash = git_hash[:len(git_hash)-1]
         app.logger.info('Git hash = ' + git_hash)
 
-        # create directories using concatenated token of git-hash and md5sum of hwdef
-        token = git_hash + "-" + md5sum
+        # create directories using concatenated token 
+        # of vehicle, board, git-hash of source, and md5sum of hwdef
+        vehicle = request.form['vehicle']
+        board = request.form['board']
+        token = vehicle + '-' + board + '-' + git_hash + '-' + md5sum
         app.logger.info('token = ' + token)
-        buildqueue_dir = os.path.join(appdir, 'buildqueue', token)
-        if not os.path.isdir(buildqueue_dir):
-            app.logger.info('Creating ' + buildqueue_dir)
-            os.mkdir(buildqueue_dir)
-        app.logger.info('Opening ' + os.path.join(buildqueue_dir, 'extra_hwdef.dat'))
-        file = open(os.path.join(buildqueue_dir, 'extra_hwdef.dat'),"w")
-        app.logger.info('Writing\n' + extra_hwdef)
-        file.write(extra_hwdef)
-        file.close()
+        outdir = os.path.join(outdir_parent, token)
         
-        # fill dictionary of variables and create json file
-        task['hwdef_md5sum'] = md5sum
-        task['git_hash'] = git_hash
-        task['token'] = token
-        task['sourcedir'] = sourcedir
-        task['extra_hwdef'] = os.path.join(buildqueue_dir, 'extra_hwdef.dat')
-        task['board'] = request.form["board"]
-        task['vehicle'] = request.form["vehicle"]
-        app.logger.info('Opening ' + os.path.join(buildqueue_dir, 'q.json'))
-        jfile = open(os.path.join(buildqueue_dir, 'q.json'), "w")
-        app.logger.info('Writing task file to ' + os.path.join(buildqueue_dir, 'q.json'))
-        jfile.write(json.dumps(task))
-        jfile.close()
+        if os.path.isdir(outdir):
+            app.logger.info('Build already exists')
+        else:
+            app.logger.info('Creating ' + outdir)
+            create_directory(outdir)
+            # create build.log
+            app.logger.info('Creating build.log')
+            build_log = open(os.path.join(outdir, 'build.log'), 'w')
+            build_log.write('Waiting for build to start...\n')
+            build_log.close()
+            # create hwdef.dat
+            app.logger.info('Opening ' + os.path.join(outdir, 'extra_hwdef.dat'))
+            file = open(os.path.join(outdir, 'extra_hwdef.dat'),'w')
+            app.logger.info('Writing\n' + extra_hwdef)
+            file.write(extra_hwdef)
+            file.close()
+            # fill dictionary of variables and create json file
+            task['hwdef_md5sum'] = md5sum
+            task['git_hash'] = git_hash
+            task['token'] = token
+            task['sourcedir'] = sourcedir
+            task['extra_hwdef'] = os.path.join(outdir, 'extra_hwdef.dat')
+            task['board'] = board
+            task['vehicle'] = vehicle
+            app.logger.info('Opening ' + os.path.join(outdir, 'q.json'))
+            jfile = open(os.path.join(outdir, 'q.json'), 'w')
+            app.logger.info('Writing task file to ' + os.path.join(outdir, 'q.json'))
+            jfile.write(json.dumps(task))
+            jfile.close()
 
         queue_lock.release()
 
-        #print(task)
-
-        apache_build_dir = "http://localhost:8080/" + token
-        apache_build_log = "http://localhost:8080/" + token + "/build.log"
+        apache_build_dir = 'http://localhost:8080/' \
+                            + os.path.join('builds', token)
+        apache_build_log = 'http://localhost:8080/' \
+                            + os.path.join('builds', token, 'build.log')
+        apache_all_builds = 'http://localhost:8080/' \
+                            + 'builds'
         app.logger.info('Rendering generate.html')
         return render_template('generate.html', 
                                 apache_build_dir=apache_build_dir, 
                                 apache_build_log=apache_build_log,
+                                apache_all_builds=apache_all_builds,
                                 token=token)
     
     except:
@@ -263,5 +266,5 @@ def home():
     app.logger.info('Rendering index.html')
     return render_template('index.html')
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     app.run()

app_test.py

@@ -1,155 +0,0 @@
-import os
-import time
-import pytest
-
-from app import app
-
-def createFile(name, size):
-    # create a file of random data
-    with open(name, 'wb') as fout:
-        fout.write(os.urandom(size)) # replace 1024 with size_kb if not unreasonably large
-
-
-@pytest.fixture
-def client():
-    app.config['TESTING'] = True
-    with app.test_client() as client:
-        yield client
-
-def test_homepage(client):
-    """Test that the homepage can be generated"""
-
-    rv = client.get('/')
-    # assert all the controls are there
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'<form action="/generate" method="post">' in rv.data
-    assert b'<input type="text" id="lat" name="lat" value="-35.363261" oninput="plotCircleCoords(true);">' in rv.data
-    assert b'<input type="text" id="long" name="long" value="149.165230" oninput="plotCircleCoords(true);">' in rv.data
-    assert b'<input type="range" id="radius" name="radius" value="100" min="1" max="400"\n                oninput="plotCircleCoords(false);">' in rv.data
-    assert b'<input type="submit" value="Generate" method="post">' in rv.data
-
-def test_badinput(client):
-    """Test bad inputs"""
-    # no input
-    rv = client.post('/generate', data=dict(
-    ), follow_redirects=True)
-
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'Error' in rv.data
-    assert b'Link To Download' not in rv.data
-
-    #partial input
-    rv = client.post('/generate', data=dict(
-        lat='-35.363261',
-        long='149.165230',
-    ), follow_redirects=True)
-
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'Error' in rv.data
-    assert b'Link To Download' not in rv.data
-
-    #bad lon/lat
-    rv = client.post('/generate', data=dict(
-        lat='I am bad data',
-        long='echo test',
-        radius='1',
-    ), follow_redirects=True)
-
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'Error' in rv.data
-    assert b'download="terrain.zip"' not in rv.data
-
-    #out of bounds lon/lat
-    rv = client.post('/generate', data=dict(
-        lat='206.56',
-        long='-400',
-        radius='1',
-    ), follow_redirects=True)
-
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'Error' in rv.data
-    assert b'download="terrain.zip"' not in rv.data
-
-def test_simplegen(client):
-    """Test that a small piece of terrain can be generated"""
-
-    rv = client.post('/generate', data=dict(
-        lat='-35.363261',
-        long='149.165230',
-        radius='1',
-    ), follow_redirects=True)
-
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'Error' not in rv.data
-    assert b'Tiles outside of +60 to -60 latitude were requested' not in rv.data
-    assert b'download="terrain.zip"' in rv.data
-
-    uuidkey = (rv.data.split(b"footer")[1][1:-2]).decode("utf-8")
-    assert uuidkey != ""
-
-    #file should be ready for download and around 2MB in size
-    rdown = client.get('/terrain/' + uuidkey + ".zip", follow_redirects=True)
-    assert b'404 Not Found' not in rdown.data
-    assert len(rdown.data) > (1*1024*1024)
-
-def test_simplegenoutside(client):
-    """Test that a small piece of terrain can be generated with partial outside +-60latitude"""
-
-    rv = client.post('/generate', data=dict(
-        lat='-59.363261',
-        long='149.165230',
-        radius='200',
-    ), follow_redirects=True)
-
-    assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-    assert b'Error' not in rv.data
-    assert b'Tiles outside of +60 to -60 latitude were requested' in rv.data
-    assert b'download="terrain.zip"' in rv.data
-
-    uuidkey = (rv.data.split(b"footer")[1][1:-2]).decode("utf-8")
-    assert uuidkey != ""
-
-    #file should be ready for download and around 2MB in size
-    rdown = client.get('/terrain/' + uuidkey + ".zip", follow_redirects=True)
-    assert b'404 Not Found' not in rdown.data
-    assert len(rdown.data) > (0.25*1024*1024)
-
-def test_multigen(client):
-    """Test that a a few small piece of terrains can be generated"""
-
-    rva = client.post('/generate', data=dict(
-        lat='-35.363261',
-        long='149.165230',
-        radius='1',
-    ), follow_redirects=True)
-    time.sleep(0.1)
-
-    rvb = client.post('/generate', data=dict(
-        lat='-35.363261',
-        long='147.165230',
-        radius='10',
-    ), follow_redirects=True)
-    time.sleep(0.1)
-
-    rvc = client.post('/generate', data=dict(
-        lat='-30.363261',
-        long='137.165230',
-        radius='100',
-    ), follow_redirects=True)
-    time.sleep(0.1)
-
-    # Assert reponse is OK and get UUID for each ter gen
-    allUuid = []
-    for rv in [rva, rvb, rvc]:
-        assert b'<title>ArduPilot Terrain Generator</title>' in rv.data
-        assert b'Error' not in rv.data
-        assert b'download="terrain.zip"' in rv.data
-        uuidkey = (rv.data.split(b"footer")[1][1:-2]).decode("utf-8")
-        assert uuidkey != ""
-        allUuid.append(uuidkey)
-
-    #files should be ready for download and around 0.7MB in size
-    for uukey in allUuid:
-        rdown = client.get('/terrain/' + uukey + ".zip", follow_redirects=True)
-        assert b'404 Not Found' not in rdown.data
-        assert len(rdown.data) > (0.7*1024*1024)

templates/generate.html

@@ -10,13 +10,13 @@
 
         <p>Build in progress...</p>
         
-        <form action="{{ apache_build_log }}">
+        <form action="{{ apache_build_log }}" target="_blank">
             <input type="submit" value="View build progress" />
         </form>
-        <form action="{{ apache_build_dir }}">
+        <form action="{{ apache_build_dir }}" target="_blank">
             <input type="submit" value="Go to build directory" />
         </form>
-        <form action="http://localhost:8080/">
+        <form action="{{ apache_all_builds }}" target="_blank">
             <input type="submit" value="See all builds" />
         </form>
         <form action="/home" method="post">

templates/index.html

@@ -52,4 +52,4 @@
 </body>
 <br />
 
-<footer>Created by Will Piper, <a href=https://github.com/willpiper24/terraingen>Source Code<a>.</footer>
+<footer>Created by Will Piper, <a href=https://github.com/ArduPilot/CustomBuild>Source Code<a>.</footer>

templates/index_curl.html

@@ -1,77 +0,0 @@
-<!doctype html>
-
-<body>
-    <div id="menu">
-        <title>ArduPilot Custom Firmware Builder</title>
-        <h1>ArduPilot Custom Firmware Builder</h1>
-
-        <p>Please select the required options for the reduced firmware build.</p>
-
-        <form action="/generate" method="post">
-            <input name="vehicle">
-            <input name="board">
-            <input name="option1">
-            <input name="option2">
-            <input name="option3">
-            <input name="option4">
-            <input name="option5">
-            <input name="option6">
-            <input name="option7">
-            <input name="option8">
-        </form>
-    </div>
-
-
-
-    <!-- <div id="menu">
-        <title>ArduPilot Custom Firmware Builder</title>
-        <h1>ArduPilot Custom Firmware Builder</h1>
-
-        <p>Please select the required options for the reduced firmware build.</p>
-
-        <form action="/generate" method="post">
-            <label for="vehicle">Choose a vehicle:</label>
-            <select name="vehicle">
-                <option value="copter">Copter</option>
-                <option value="plane">Plane</option>
-                <option value="rover">Rover</option>
-                <option value="sub">Sub</option>
-            </select>
-            <br><br>
-            <label for="board">Choose a board:</label>
-            <select name="board">
-                <option value="BeastF7">BeastF7</option>
-                <option value="BeastH7">BeastH7</option>
-            </select>
-            <br><br>
-            <input type="checkbox" name="option1" value="define HAL_NAVEKF2_AVAILABLE 1">
-            <input type="hidden" name="option1" value="define HAL_NAVEKF2_AVAILABLE 0"> EKF2
-            <br>
-            <input type="checkbox" name="option2" value="define HAL_NAVEKF3_AVAILABLE 1">
-            <input type="hidden" name="option2" value="define HAL_NAVEKF3_AVAILABLE 0"> EKF3
-            <br>
-            <input type="checkbox" name="option3" value="define HAL_WITH_DSP 1">
-            <input type="hidden" name="option3" value="define HAL_WITH_DSP 0"> HAL_WITH_DSP
-            <br>
-            <input type="checkbox" name="option4" value="define HAL_SPRAYER_ENABLED 1">
-            <input type="hidden" name="option4" value="define HAL_SPRAYER_ENABLED 0"> HAL_SPRAYER_ENABLED
-            <br>
-            <input type="checkbox" name="option5" value="define HAL_PARACHUTE_ENABLED 1">
-            <input type="hidden" name="option5" value="define HAL_PARACHUTE_ENABLED 0"> HAL_PARACHUTE_ENABLED
-            <br>
-            <input type="checkbox" name="option6" value="define HAL_MOUNT_ENABLED 1">
-            <input type="hidden" name="option6" value="define HAL_MOUNT_ENABLED 0"> HAL_MOUNT_ENABLED
-            <br>
-            <input type="checkbox" name="option7" value="define HAL_HOTT_TELEM_ENABLED 1">
-            <input type="hidden" name="option7" value="define HAL_HOTT_TELEM_ENABLED 0"> HAL_HOTT_TELEM_ENABLED
-            <br>
-            <input type="checkbox" name="option8" value="define HAL_BATTMON_FUEL_ENABLE 1">
-            <input type="hidden" name="option8" value="define HAL_BATTMON_FUEL_ENABLE 0"> HAL_BATTMON_FUEL_ENABLE
-            <br><br>
-            <input type="submit" value="Generate" method="post">
-        </form>
-    </div> -->
-</body>
-<br />
-
-<footer>Created by Will Piper, <a href=https://github.com/willpiper24/terraingen>Source Code<a>.</footer>