Added restart and compression

offline_gen.py

@@ -10,15 +10,47 @@ import os
 from multiprocessing.pool import ThreadPool
 import argparse
 import time
+import gzip
+import shutil
 
 from MAVProxy.modules.mavproxy_map import srtm
 from terrain_gen import create_degree
 
 def worker(downloader, lat, long, targetFolder, startedTiles, totTiles):
-    create_degree(downloader, lat, long, targetFolder, 100)
+    # only create if the output file does not exists
+    if os.path.exists(datafile(lat, long, targetFolder) + '.gz'):
+        print("Skipping existing compressed tile {0} of {1} ({2:.3f}%)".format(startedTiles, totTiles, ((startedTiles)/totTiles)*100))
+        return
+
+    if not os.path.exists(datafile(lat, long, targetFolder)):
+        create_degree(downloader, lat, long, targetFolder, 100)
+        print("Created tile {0} of {1}".format(startedTiles, totTiles))
+    else:
+        print("Skipping existing tile {0} of {1} ({2:.3f}%)".format(startedTiles, totTiles, ((startedTiles)/totTiles)*100))
+
+    # and compress
+    with open(datafile(lat, long, targetFolder), 'rb') as f_in:
+        with gzip.open(datafile(lat, long, targetFolder) + '.gz', 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+    os.remove(datafile(lat, long, targetFolder))
+
     print("Done tile {0} of {1} ({2:.3f}%)".format(startedTiles, totTiles, ((startedTiles)/totTiles)*100))
     print("Folder is {0:.0f}Mb in size".format(get_size(targetFolder)/(1024*1024)))
 
+def datafile(lat, lon, folder):
+    if lat < 0:
+        NS = 'S'
+    else:
+        NS = 'N'
+    if lon < 0:
+        EW = 'W'
+    else:
+        EW = 'E'
+    name = folder + "/%c%02u%c%03u.DAT" % (NS, min(abs(int(lat)), 99),
+                                    EW, min(abs(int(lon)), 999))
+
+    return name
+
 def get_size(start_path = '.'):
     total_size = 0
     for dirpath, dirnames, filenames in os.walk(start_path):
@@ -26,7 +58,12 @@ def get_size(start_path = '.'):
             fp = os.path.join(dirpath, f)
             # skip if it is symbolic link
             if not os.path.islink(fp):
-                total_size += os.path.getsize(fp)
+                # other threads may be moving files whilst we're getting folder
+                # size here
+                try:
+                    total_size += os.path.getsize(fp)
+                except FileNotFoundError:
+                    pass
 
     return total_size