Changes to packaging

.github/workflows/build.yml

@@ -53,13 +53,13 @@ jobs:
 
     - name: Build
       run: |
-        make autoortho_win.exe
+        make exe
         make autoortho_win.zip ZIP="7z a"
 
     - name: Save artifact
       uses: actions/upload-artifact@v3
       with:
         name: winbin
         path: |
-          autoortho_win.exe
+          AutoOrtho_*.exe
           autoortho_win.zip

.gitignore

@@ -6,3 +6,10 @@ out.dds
 /cache
 **/libjpeg-turbo-gcc64
 tags
+__main__.*
+autoortho.*
+build/
+cache/
+downloads/
+release/
+*.bin

Makefile

@@ -1,4 +1,5 @@
 ZIP?=zip
+VERSION?=0.0.0
 
 autoortho.pyz:
 	mkdir -p build/autoortho
@@ -24,7 +25,7 @@ bin:
 		--onefile \
 		./autoortho/__main__.py -o autoortho_lin.bin
 
-autoortho_win.exe:
+_autoortho_win.exe:
 	python3 -m nuitka --verbose --verbose-output=nuitka.log \
 		--mingw64 \
 		--disable-ccache \
@@ -53,6 +54,13 @@ __main__.dist:
 		--include-data-dir=./autoortho/imgs=imgs \
 		--standalone \
 		./autoortho/__main__.py -o autoortho_win.exe
+
+exe: AutoOrtho_$(VERSION).exe
+AutoOrtho_$(VERSION).exe: __main__.dist
+	cp autoortho/imgs/ao-icon.ico .
+	makensis -DPRODUCT_VERSION=$(VERSION) installer.nsi
+	mv AutoOrtho.exe $@
+
 autoortho_win.zip: __main__.dist
 	mv __main__.dist autoortho_release
 	$(ZIP) $@ autoortho_release

autoortho/__main__.py

@@ -4,7 +4,7 @@
 import sys
 import logging
 import logging.handlers
-from aoconfig import CFG
+from .aoconfig import CFG
 
 def setuplogs():
     log_dir = os.path.join(os.path.expanduser("~"), ".autoortho-data", "logs")
@@ -25,7 +25,7 @@ def setuplogs():
             ]
     )
 
-import autoortho
+from . import autoortho
 
 if __name__ == "__main__":
     setuplogs()

autoortho/aoconfig.py

@@ -12,15 +12,18 @@
 import configparser
 import platform
 
+import importlib.resources
+import inspect
+
 import logging
 log = logging.getLogger(__name__)
 
 import PySimpleGUI as sg
 
-import downloader
-
-CUR_PATH = os.path.dirname(os.path.realpath(__file__))
+from . import downloader
 
+#CUR_PATH = os.path.dirname(os.path.realpath(__file__))
+CUR_PATH = str(importlib.resources.files(__package__))
 
 class SectionParser:
     true = ['true','1', 'yes', 'on']
@@ -71,7 +74,9 @@ def __init__(self, cfg):
             sg.theme('DarkAmber')
 
         self.scenery_q = queue.Queue()
-
+
+        CUR_PATH = str(importlib.resources.files(__package__))
+        #log.info(CUR_PATH)
         if platform.system() == 'Windows':
             self.icon_path =os.path.join(CUR_PATH, 'imgs', 'ao-icon.ico')
         else:

autoortho/aoimage/AoImage.py

@@ -8,6 +8,9 @@
 import logging
 log = logging.getLogger(__name__)
 
+import importlib.resources
+CUR_PATH = importlib.resources.files(__package__).joinpath('')
+
 class AOImageException(Exception):
     pass
 
@@ -146,9 +149,9 @@ def open(filename):
 
 # init code
 if platform.system().lower() == 'linux':
-    _aoi_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'aoimage.so')
+    _aoi_path = os.path.join(CUR_PATH,'aoimage.so')
 elif platform.system().lower() == 'windows':
-    _aoi_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'aoimage.dll')
+    _aoi_path = os.path.join(CUR_PATH,'aoimage.dll')
 else:
     log.error("System is not supported")
     exit()

autoortho/aostats.py

@@ -3,7 +3,7 @@
 import threading
 import collections
 
-from aoconfig import CFG
+from .aoconfig import CFG
 import logging
 log = logging.getLogger(__name__)
 

autoortho/autoortho.py

@@ -8,11 +8,11 @@
 import argparse
 import threading
 
-import aoconfig
-import aostats
-import winsetup
+from . import aoconfig
+from . import aostats
+from . import winsetup
 
-import flighttrack
+from . import flighttrack
 #import multiprocessing
 
 import logging
@@ -40,7 +40,7 @@ def run(root, mountpoint, threading=True):
             mountpoint = os.path.expanduser(mountpoint)
             winsetup.setup_dokan_mount(mountpoint) 
             log.info(f"AutoOrtho:  root: {root}  mountpoint: {mountpoint}")
-            import autoortho_fuse
+            from . import autoortho_fuse
             autoortho_fuse.run(
                     autoortho_fuse.AutoOrtho(root), 
                     mountpoint, 
@@ -52,7 +52,7 @@ def run(root, mountpoint, threading=True):
             mountpoint = os.path.expanduser(mountpoint)
             winsetup.setup_winfsp_mount(mountpoint) 
             log.info(f"AutoOrtho:  root: {root}  mountpoint: {mountpoint}")
-            import autoortho_fuse
+            from . import autoortho_fuse
             autoortho_fuse.run(
                     autoortho_fuse.AutoOrtho(root), 
                     mountpoint, 
@@ -70,7 +70,7 @@ def run(root, mountpoint, threading=True):
             sys.exit(1)
 
         log.info(f"AutoOrtho:  root: {root}  mountpoint: {mountpoint}")
-        import autoortho_fuse
+        from . import autoortho_fuse
         autoortho_fuse.run(
                 autoortho_fuse.AutoOrtho(root),
                 mountpoint, 
@@ -134,7 +134,7 @@ def main():
     #if CFG.cache.clean_on_start:
     #    aoconfig.clean_cache(CFG.paths.cache_dir, int(float(CFG.cache.file_cache_size)))
 
-    import flighttrack
+    from . import flighttrack
     ftrack = threading.Thread(
         target=flighttrack.run,
         daemon=True

autoortho/autoortho_fuse.py

@@ -17,20 +17,20 @@
 import threading
 import itertools
 
-import flighttrack
+from . import flighttrack
 
 from functools import wraps, lru_cache
 
-from aoconfig import CFG
+from .aoconfig import CFG
 import logging
 log = logging.getLogger(__name__)
 
 #from fuse import FUSE, FuseOSError, Operations, fuse_get_context
 from refuse.high import FUSE, FuseOSError, Operations, fuse_get_context, fuse_exit, _libfuse
 
-import getortho
+from . import getortho
 
-from xp_udp import DecodePacket, RequestDataRefs
+from .xp_udp import DecodePacket, RequestDataRefs
 import socket
 
 #from memory_profiler import profile
@@ -305,9 +305,6 @@ def open(self, path, flags):
         full_path = self._full_path(path)
         log.debug(f"OPEN: FULL PATH: {full_path}")
 
-        dsf_m = self.dsf_re.match(path)
-        if dsf_m:
-            log.info(f"OPEN: Detected DSF open: {path}")
         #dsf_m = self.dsf_re.match(path)
         #ter_m = self.ter_re.match(path)
         dds_m = self.dds_re.match(path)
@@ -316,8 +313,18 @@ def open(self, path, flags):
             row = int(row)
             col = int(col)
             zoom = int(zoom)
-            t = self.tc._open_tile(row, col, maptype, zoom) 
-        elif platform.system() == 'Windows':
+            t = self.tc._open_tile(row, col, maptype, zoom)
+            return 0
+
+
+        dsf_m = self.dsf_re.match(path)
+        if dsf_m:
+            log.info(f"OPEN: Detected DSF open: {path}")
+
+        if not os.path.exists(full_path):
+            log.warning(f"Requested path {full_path} not found!")
+
+        if platform.system() == 'Windows':
             h = os.open(full_path, flags|os.O_BINARY)
         else:
             h = os.open(full_path, flags)

autoortho/flighttrack.py

@@ -5,16 +5,16 @@
 import json
 import socket
 import threading
-from aoconfig import CFG
+from .aoconfig import CFG
 import logging
 log = logging.getLogger(__name__)
 
 from flask import Flask, render_template, url_for, request, jsonify
 from flask_socketio import SocketIO, send, emit
 
-from xp_udp import DecodePacket, RequestDataRefs
+from .xp_udp import DecodePacket, RequestDataRefs
 
-from aostats import STATS
+from .aostats import STATS
 #STATS = {'count': 71036, 'chunk_hit': 66094, 'mm_counts': {0: 19, 1: 39, 2: 97, 3: 294, 4: 2982}, 'mm_averages': {0: 0.56, 1: 0.14, 2: 0.04, 3: 0.01, 4: 0.0}, 'chunk_miss': 4942, 'bytes_dl': 65977757}
 
 RUNNING=True

autoortho/getortho.py

@@ -17,15 +17,13 @@
 from functools import wraps, lru_cache
 from pathlib import Path
 
-import pydds
-
-
 import psutil
 from PIL import Image
-from aoimage import AoImage
 
-from aoconfig import CFG
-from aostats import STATS, StatTracker, set_stat, inc_stat
+from . import pydds
+from .aoimage import AoImage
+from .aoconfig import CFG
+from .aostats import STATS, StatTracker, set_stat, inc_stat
 
 MEMTRACE = False
 
@@ -128,11 +126,13 @@ def worker(self, idx):
 
             try:
                 if not self.get(obj, *args, **kwargs):
+                    inc_stat('chunk_get_fail')
                     log.warning(f"Failed getting: {obj} {args} {kwargs}, re-submit.")
-                    self.submit(obj, *args, **kwargs)
+                    #self.submit(obj, *args, **kwargs)
             except Exception as err:
+                inc_stat('chunk_get_error')
                 log.error(f"ERROR {err} getting: {obj} {args} {kwargs}, re-submit.")
-                self.submit(obj, *args, **kwargs)
+                #self.submit(obj, *args, **kwargs)
 
     def get(obj, *args, **kwargs):
         raise NotImplementedError
@@ -191,6 +191,7 @@ class Chunk(object):
     url = None
 
     serverlist=['a','b','c','d']
+    serverlist2=['t0','t1','t2','t3']
 
     def __init__(self, col, row, maptype, zoom, priority=0, cache_dir='.cache'):
         self.col = col
@@ -258,15 +259,18 @@ def get(self, idx=0):
             self.startime = time.time()
 
         server_num = idx%(len(self.serverlist))
+        server_num2 = idx%(len(self.serverlist2))
         server = self.serverlist[server_num]
+        server2 = self.serverlist[server_num2]
         quadkey = _gtile_to_quadkey(self.col, self.row, self.zoom)
 
         # Hack override maptype
         #maptype = "ARC"
 
         MAPTYPES = {
             "EOX": f"https://{server}.s2maps-tiles.eu/wmts/?layer={MAPID}&style=default&tilematrixset={MATRIXSET}&Service=WMTS&Request=GetTile&Version=1.0.0&Format=image%2Fjpeg&TileMatrix={self.zoom}&TileCol={self.col}&TileRow={self.row}",
-            "BI": f"http://r{server_num}.ortho.tiles.virtualearth.net/tiles/a{quadkey}.jpeg?g=136",
+            "BI2": f"http://r{server_num}.ortho.tiles.virtualearth.net/tiles/a{quadkey}.jpeg?g=136",
+            "BI":f"http://ecn.t{server_num}.tiles.virtualearth.net/tiles/a{quadkey}.jpeg?g=13716",
             #"GO2": f"http://khms{server_num}.google.com/kh/v=934?x={self.col}&y={self.row}&z={self.zoom}",
             "ARC": f"http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{self.zoom}/{self.row}/{self.col}",
             "NAIP": f"http://naip.maptiles.arcgis.com/arcgis/rest/services/NAIP/MapServer/tile/{self.zoom}/{self.row}/{self.col}",
@@ -629,6 +633,7 @@ def get_bytes(self, offset, length):
 
         return True
 
+    @locked
     def read_dds_bytes(self, offset, length):
         log.debug(f"READ DDS BYTES: {offset} {length}")
 
@@ -800,6 +805,7 @@ def get_img(self, mipmap, startrow=0, endrow=None, maxwait=5, min_zoom=None):
         # Return image along with mipmap and zoom level this was created at
         return new_im
 
+    @locked
     def get_best_chunk(self, col, row, mm, zoom):
         for i in range(mm+1, 5):
 

autoortho/pydds.py

@@ -7,28 +7,34 @@
 from binascii import hexlify
 from ctypes import *
 #from PIL import Image
-from aoimage import AoImage as Image
+import inspect
+
+from .aoimage import AoImage as Image
+from .aoconfig import CFG
 
 import platform
 import threading
 
 #from functools import lru_cache, cache
 
 #from memory_profiler import profile
-from aoconfig import CFG
 
 import logging
 log = logging.getLogger(__name__)
 
+import importlib.resources
+#CUR_PATH = os.path.dirname(os.path.realpath(__file__))
+CUR_PATH = str(importlib.resources.files(__package__))
+
 #_stb = CDLL("/usr/lib/x86_64-linux-gnu/libstb.so")
 if platform.system().lower() == 'linux':
     print("Linux detected")
-    _stb_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'lib','linux','lib_stb_dxt.so')
-    _ispc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'lib','linux','libispc_texcomp.so')
+    _stb_path = os.path.join(CUR_PATH,'lib','linux','lib_stb_dxt.so')
+    _ispc_path = os.path.join(CUR_PATH,'lib','linux','libispc_texcomp.so')
 elif platform.system().lower() == 'windows':
     print("Windows detected")
-    _stb_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'lib','windows','stb_dxt.dll')
-    _ispc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'lib','windows','ispc_texcomp.dll')
+    _stb_path = os.path.join(CUR_PATH,'lib','windows','stb_dxt.dll')
+    _ispc_path = os.path.join(CUR_PATH,'lib','windows','ispc_texcomp.dll')
 else:
     print("System is not supported")
     exit()

autoortho/test_downloader.py

@@ -6,7 +6,7 @@
 import pytest
 import platform
 
-import downloader
+from . import downloader
 downloader.TESTMODE = True
 
 def test_setup(tmpdir):