Now using pre-processed terrain

app.py

@@ -5,6 +5,7 @@ import queue
 import os
 import zipfile
 import shutil
+import sys
 
 from flask import Flask
 from flask import render_template
@@ -12,8 +13,7 @@ from flask import request
 from flask import json, jsonify
 
 from MAVProxy.modules.mavproxy_map import srtm
-
-from terrain_gen import makeTerrain
+from terrain_gen import add_offset
 
 # The output folder for all zipped terrain requests
 app = Flask(__name__,static_url_path='/terrain', static_folder='outputTer',)
@@ -21,71 +21,35 @@ app = Flask(__name__,static_url_path='/terrain', static_folder='outputTer',)
 def clamp(n, smallest, largest):
     return max(smallest, min(n, largest))
 
-class TerGenThread(threading.Thread):
-    # This is the terrain generator. It will check the queue 
-    # for new requests.
-    def __init__(self):
-        threading.Thread.__init__(self)
-        self.event = threading.Event()
-        # SRTM downloader. Single instance passed to terrain generator
-        self.downloader = srtm.SRTMDownloader(debug=False, cachedir='./srtmcache')
-        self.downloader.loadFileList()
-        self.terStats = {}
-        self.terQ = queue.Queue()
-
-    def addStatus(self, uuidkey, value):
-        self.terStats[uuidkey] = value
-
-    def run(self):
-        print("Starting Terrain Generator")
-        while not self.event.is_set():
-            time.sleep(0.01)
-            if not self.terQ.empty():
-                (radius, lat, lon, spacing, uuidkey, outfolder) = self.terQ.get()
-                print("Starting request: " + str(uuidkey))
-                self.terStats[uuidkey] = "Processing"
-                # generate terrain
-                makeTerrain(self.downloader, radius, lat, lon, spacing, uuidkey, outfolder)
-                
-                self.terStats[uuidkey] = "Compressing"
-                #compress into single file for user
-                folderthis = os.path.join(os.getcwd(), outfolder + "-tmp", uuidkey)
-                zipthis = os.path.join(os.getcwd(), outfolder, uuidkey + '.zip')
-                terrain_zip = zipfile.ZipFile(zipthis, 'w')
-                 
-                print("At Compress ")
-                #numpercent  = numpercent * 1.1
-                for folder, subfolders, files in os.walk(folderthis):
-                    for file in files:
-                        terrain_zip.write(os.path.join(folder, file), file, compress_type = zipfile.ZIP_DEFLATED)
-                 
-                terrain_zip.close()
-
-                #remove old folder
-                try:
-                    shutil.rmtree(folderthis)
-                except OSError as e:
-                    print("Error: %s : %s" % (folderthis, e.strerror))
-
-                print("At Done")
-
-                del self.terStats[uuidkey]
-
-        print("Exiting Terrain Generator")
-
-# Terrain generator checker
-x = TerGenThread()
-x.start()
-
-def queueStatus():
-    return len(x.terStats)
-
-def wholeStat():
-    return str(x.terStats)
-
-def shutdown():
-    x.event.set()
-    x.join()
+def getDatFile(lat, lon):
+    if lat < 0:
+        NS = 'S'
+    else:
+        NS = 'N'
+    if lon < 0:
+        EW = 'W'
+    else:
+        EW = 'E'
+    return "%c%02u%c%03u.DAT" % (NS, min(abs(int(lat)), 99),
+                                    EW, min(abs(int(lon)), 999))
+
+def compressFiles(fileList, uuidkey, outfolder):
+    # create a zip file comprised of dat tiles
+    zipthis = os.path.join(os.getcwd(), outfolder, uuidkey + '.zip')
+    terrain_zip = zipfile.ZipFile(zipthis, 'w')
+     
+    try:
+        for fileSingle in fileList:
+            # terrain_zip.write(os.path.join(folder, file), file, compress_type = zipfile.ZIP_DEFLATED)
+            terrain_zip.write(fileSingle, os.path.basename(fileSingle), compress_type = zipfile.ZIP_DEFLATED)
+    except:
+        print("Unexpected error:", sys.exc_info()[0])
+        terrain_zip.close()
+        return False
+        
+     
+    terrain_zip.close()
+    return True
 
 @app.route('/')
 def index():
@@ -112,23 +76,36 @@ def generate():
         # UUID for this terrain generation
         uuidkey = str(uuid.uuid1())
 
-        # Add this job to the processing queue
-        x.terQ.put((radius, lat, lon, 100, uuidkey, 'outputTer'))
-        #x.terStats[uuidkey] = "In Queue, pos={0}".format(terQ.qsize())
-        x.addStatus(uuidkey, "In Queue, pos={0}".format(x.terQ.qsize()))
-
-        return render_template('generate.html', urlkey="/terrain/" + uuidkey + ".zip", waittime=x.terQ.qsize()+1, uuidkey=uuidkey)
+        # get a list of files required to cover area
+        filelist = []
+        done = set()
+        for dx in range(-radius, radius):
+            for dy in range(-radius, radius):
+                (lat2,lon2) = add_offset(lat*1e7, lon*1e7, dx*1000.0, dy*1000.0)
+                lat_int = int(round(lat2 * 1.0e-7))
+                lon_int = int(round(lon2 * 1.0e-7))
+                tag = (lat_int, lon_int)
+                if tag in done:
+                    #numpercent += 1
+                    continue
+                done.add(tag)
+                #create_degree(downloader, lat_int, lon_int, folderthis, spacing)
+                filelist.append(os.path.join(os.getcwd(), "processedTerrain", getDatFile(lat_int, lon_int)))
+
+        #create_degree(downloader, lat, lon, folderthis, spacing)
+        #filelist.append(getDatFile(lat, lon))
+        print(filelist)
+
+        #compress
+        success = compressFiles(filelist, uuidkey, 'outputTer')
+        
+        if success:
+            print("Generated " + "/terrain/" + uuidkey + ".zip")
+            return render_template('generate.html', urlkey="/terrain/" + uuidkey + ".zip", uuidkey=uuidkey)
+        else:
+            print("Failed " + "/terrain/" + uuidkey + ".zip")
+            return render_template('generate.html', error = "Processing terrain", uuidkey=uuidkey)
     else:
         print("Bad get")
         return render_template('generate.html', error = "Need to use POST, not GET")
 
-@app.route('/status/<uuid:uuidkey>')
-def status(uuidkey):
-    if not uuidkey:
-        return jsonify(status = 'Error: incorrect UUID')
-    elif str(uuidkey) in x.terStats:
-        return jsonify(status = 'success', data = str(x.terStats[str(uuidkey)]))
-    elif os.path.exists(os.path.join('.', 'outputTer', str(uuidkey) + ".zip")):
-        return jsonify(status = 'success', data = "ready")
-    else:
-        return jsonify(status = 'Error: bad UUID ' + str(os.path.join('.', 'outputTer', str(uuidkey) + ".zip")))

app_test.py

@@ -4,13 +4,28 @@ import time
 import pytest
 import uuid
 
-from app import app, queueStatus, shutdown, wholeStat
+from app import app
+
+def createFile(name, size):
+    # create a file of random data
+    with open(name, 'wb') as fout:
+        fout.write(os.urandom(size)) # replace 1024 with size_kb if not unreasonably large
 
 
 @pytest.fixture
 def client():
     app.config['TESTING'] = True
 
+    # create fake pre-gen terrain files if they don't exist
+    preGen = ['S35E149.DAT', 'S35E147.DAT', 'S30E137.DAT', 'S31E136.DAT', 'S31E137.DAT', 'S31E138.DAT',
+              'S30E136.DAT', 'S30E137.DAT', 'S30E138.DAT', 'S29E136.DAT', 'S29E137.DAT', 'S29E138.DAT']
+    for fileSingle in preGen:
+        full = os.path.join(os.getcwd(), "processedTerrain", fileSingle)
+        print(full)
+        if not os.path.exists(full):
+            print("Making fake file: " + full)
+            createFile(full, 1024 * 1024)
+
     with app.test_client() as client:
         #with app.app_context():
         #    app.init_db()
@@ -30,18 +45,6 @@ def test_homepage(client):
     assert b'<input type="number" id="radius" name="radius" value="100" min="1" max="400">' in rv.data
     assert b'<input type="submit" value="Submit" method="post">' in rv.data
 
-def test_status(client):
-    """Test bad inputs to status page"""
-    uuidkey = str(uuid.uuid1())
-    rc = client.get('/status/' + uuidkey, follow_redirects=True)
-    assert b'Error: bad UUID' in rc.data
-
-    rc = client.get('/status/', follow_redirects=True)
-    assert b'404 Not Found' in rc.data
-
-    rc = client.get('/status/notauuid123' + uuidkey, follow_redirects=True)
-    assert b'404 Not Found' in rc.data
-
 def test_badinput(client):
     """Test bad inputs"""
     # no input
@@ -71,7 +74,7 @@ def test_badinput(client):
 
     assert b'<title>AP Terrain Generator</title>' in rv.data
     assert b'Error' in rv.data
-    assert b'Link To Download' not in rv.data
+    assert b'download="terrain.zip"' not in rv.data
 
     #out of bounds lon/lat
     rv = client.post('/generate', data=dict(
@@ -82,7 +85,7 @@ def test_badinput(client):
 
     assert b'<title>AP Terrain Generator</title>' in rv.data
     assert b'Error' in rv.data
-    assert b'Link To Download' not in rv.data
+    assert b'download="terrain.zip"' not in rv.data
 
 def test_simplegen(client):
     """Test that a small piece of terrain can be generated"""
@@ -95,22 +98,13 @@ def test_simplegen(client):
 
     assert b'<title>AP Terrain Generator</title>' in rv.data
     assert b'Error' not in rv.data
-    assert b'Link To Download' in rv.data
+    assert b'download="terrain.zip"' in rv.data
 
     uuidkey = (rv.data.split(b"footer")[1][1:-2]).decode("utf-8") 
     assert uuidkey != ""
 
-    #wait for generator to complete, up to 30 seconds
-    startime = time.time()
-    while True:
-        time.sleep(0.5)
-        if time.time() - startime > 30:
-            assert False
-            break
-        else:
-            rc = client.get('/status/' + uuidkey, follow_redirects=True)
-            if "ready" in rc.data.decode("utf-8") :
-                break
+    #wait for generator to complete, up to 1 second
+    time.sleep(1)
 
     #file should be ready for download and around 2MB in size
     rdown = client.get('/terrain/' + uuidkey + ".zip", follow_redirects=True)
@@ -131,14 +125,14 @@ def test_multigen(client):
     rvb = client.post('/generate', data=dict(
         lat='-35.363261',
         long='147.165230',
-        radius='1',
+        radius='10',
     ), follow_redirects=True)
     time.sleep(0.1)
 
     rvc = client.post('/generate', data=dict(
         lat='-30.363261',
         long='137.165230',
-        radius='1',
+        radius='100',
     ), follow_redirects=True)
     time.sleep(0.1)
 
@@ -147,31 +141,18 @@ def test_multigen(client):
     for rv in [rva, rvb, rvc]:
         assert b'<title>AP Terrain Generator</title>' in rv.data
         assert b'Error' not in rv.data
-        assert b'Link To Download' in rv.data
+        assert b'download="terrain.zip"' in rv.data
         uuidkey = (rv.data.split(b"footer")[1][1:-2]).decode("utf-8") 
         assert uuidkey != ""
         allUuid.append(uuidkey)
 
-    #wait for generator to complete, up to 50 seconds
-    startime = time.time()
-    allUuidComplete = []
-    while len(allUuid) != len(allUuidComplete):
-        time.sleep(1)
-        if time.time() - startime > 120:
-            break
-        else:
-            # check if done
-            for uukey in allUuid:
-                rcc = client.get('/status/' + uukey, follow_redirects=True)
-                if "ready" in rcc.data.decode("utf-8") and (uukey not in allUuidComplete):
-                    allUuidComplete.append(uukey)
+    #wait for generator to complete, up to 1 second
+    time.sleep(1)
 
     #files should be ready for download and around 2MB in size
     for uukey in allUuid:
         rdown = client.get('/terrain/' + uukey + ".zip", follow_redirects=True)
         assert len(rdown.data) > (0.7*1024*1024)
 
-    print(wholeStat())
-    shutdown()
 
 

templates/generate.html

@@ -1,12 +1,11 @@
 <!doctype html>
 <title>AP Terrain Generator</title>
 <h1>AP Terrain Generator</h1>
-<p>File is being processed. This may take up to {{ waittime }} minutes.</p>
 
 {% if error %}
   <p>Error: {{ error }}!</p>
 {% else %}
-  <p>When finished, you can download from: <a href="{{ urlkey }}" download="terrain.zip">Link To Download</a></p>
+  <p>Terrain Generation complete. You can download from: <a href="{{ urlkey }}" download="terrain.zip">here</a>.</p>
   <p>This should be unzipped to the autopilot's SD card, within in the "terrain" folder.</p>
 {% endif %}