App: update to use preprocessed files

app.py

@@ -1,27 +1,25 @@
 import uuid
-import threading
-import time
-import queue
 import os
 import zipfile
 import shutil
 import sys
+import urllib.request
+import gzip
 
 from flask import Flask
 from flask import render_template
 from flask import request
-from flask import json, jsonify
 
-from MAVProxy.modules.mavproxy_map import srtm
 from terrain_gen import add_offset
 
-# The output folder for all zipped terrain requests
-app = Flask(__name__,static_url_path='/terrain', static_folder='outputTer',)
+# The output folder for all gzipped terrain requests
+app = Flask(__name__, static_url_path='/terrain', static_folder='outputTer',)
 
 def clamp(n, smallest, largest):
     return max(smallest, min(n, largest))
 
 def getDatFile(lat, lon):
+    '''Get file'''
     if lat < 0:
         NS = 'S'
     else:
@@ -30,32 +28,54 @@ def getDatFile(lat, lon):
         EW = 'W'
     else:
         EW = 'E'
-    return "%c%02u%c%03u.DAT" % (NS, min(abs(int(lat)), 99),
-                                    EW, min(abs(int(lon)), 999))
+    return "%c%02u%c%03u.DAT.gz" % (NS, min(abs(int(lat)), 99), EW, min(abs(int(lon)), 999))
 
 def compressFiles(fileList, uuidkey, outfolder):
-    # create a zip file comprised of dat tiles
+    # create a zip file comprised of dat.gz tiles
     zipthis = os.path.join(os.getcwd(), outfolder, uuidkey + '.zip')
-    terrain_zip = zipfile.ZipFile(zipthis, 'w')
-     
+    foldertmp = os.path.join(os.getcwd(), outfolder + "-tmp", uuidkey)
+
+    # create tmp and output dirs if needed
+    os.makedirs(foldertmp)
+    try:
+        os.makedirs(os.path.join(os.getcwd(), outfolder))
+    except OSError:
+        pass
+
     try:
-        for fileSingle in fileList:
-            # terrain_zip.write(os.path.join(folder, file), file, compress_type = zipfile.ZIP_DEFLATED)
-            terrain_zip.write(fileSingle, os.path.basename(fileSingle), compress_type = zipfile.ZIP_DEFLATED)
-    except:
-        print("Unexpected error:", sys.exc_info()[0])
-        terrain_zip.close()
+        with zipfile.ZipFile(zipthis, 'w') as terrain_zip:
+            for fn in fileList:
+                if not os.path.exists(fn):
+                    #download if required
+                    g = urllib.request.urlopen('https://firmware.ardupilot.org/terrain/terraingen/processedTerrain/' + os.path.basename(fn))
+                    print("Downloaded " + os.path.basename(fn))
+                    with open(fn, 'b+w') as f:
+                        f.write(g.read())
+
+                # need to decompress file
+                datfile = os.path.join(foldertmp, os.path.basename(fn)[:-3])
+                with gzip.open(fn, 'r') as f_in, open(datfile, 'wb') as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+                    print("Decomp " + os.path.basename(fn))
+
+                # and add file to zip
+                terrain_zip.write(datfile, os.path.basename(datfile), compress_type = zipfile.ZIP_DEFLATED)
+
+            # remove cache (decompressed files)
+            shutil.rmtree(foldertmp)
+    except Exception as ex:
+        print("Unexpected error: {0}".format(ex))
         return False
         
      
-    terrain_zip.close()
+    #terrain_zip.close()
     return True
 
 @app.route('/')
 def index():
     return render_template('index.html')
 
-@app.route('/generate', methods = ['GET', 'POST'])
+@app.route('/generate', methods=['GET', 'POST'])
 def generate():
     if request.method == 'POST':
         # parse and sanitise the input
@@ -81,21 +101,23 @@ def generate():
         done = set()
         for dx in range(-radius, radius):
             for dy in range(-radius, radius):
-                (lat2,lon2) = add_offset(lat*1e7, lon*1e7, dx*1000.0, dy*1000.0)
+                (lat2, lon2) = add_offset(lat*1e7, lon*1e7, dx*1000.0, dy*1000.0)
                 lat_int = int(round(lat2 * 1.0e-7))
                 lon_int = int(round(lon2 * 1.0e-7))
                 tag = (lat_int, lon_int)
                 if tag in done:
-                    #numpercent += 1
                     continue
                 done.add(tag)
                 #create_degree(downloader, lat_int, lon_int, folderthis, spacing)
                 filelist.append(os.path.join(os.getcwd(), "processedTerrain", getDatFile(lat_int, lon_int)))
 
         #create_degree(downloader, lat, lon, folderthis, spacing)
-        #filelist.append(getDatFile(lat, lon))
+        filelist.append(os.path.join(os.getcwd(), "processedTerrain", getDatFile(lat_int, lon_int)))
+        # remove duplicates
+        filelist = list(dict.fromkeys(filelist))
         print(filelist)
 
+
         #compress
         success = compressFiles(filelist, uuidkey, 'outputTer')
         
@@ -104,8 +126,8 @@ def generate():
             return render_template('generate.html', urlkey="/terrain/" + uuidkey + ".zip", uuidkey=uuidkey)
         else:
             print("Failed " + "/terrain/" + uuidkey + ".zip")
-            return render_template('generate.html', error = "Processing terrain", uuidkey=uuidkey)
+            return render_template('generate.html', error="Cannot generate terrain", uuidkey=uuidkey)
     else:
         print("Bad get")
-        return render_template('generate.html', error = "Need to use POST, not GET")
+        return render_template('generate.html', error="Need to use POST, not GET")