From 850adc922138585332b304caeb531c64f4db3bba Mon Sep 17 00:00:00 2001 From: Sean Sall Date: Tue, 9 Apr 2019 16:08:20 +0000 Subject: [PATCH 1/6] Add dockerfile setup --- .gitignore | 6 +- README.md | 42 +++++++-- setup/Dockerfile | 40 ++++++++ setup/Dockerfile.base | 42 +++++++++ setup/build_docker_images.sh | 156 +++++++++++++++++++++++++++++++ setup/environment_cpu.yml | 174 +++++++++++++++++++++++++++++++++++ 6 files changed, 449 insertions(+), 11 deletions(-) create mode 100644 setup/Dockerfile create mode 100644 setup/Dockerfile.base create mode 100644 setup/build_docker_images.sh create mode 100644 setup/environment_cpu.yml diff --git a/.gitignore b/.gitignore index 1fcca43..5c064ea 100644 --- a/.gitignore +++ b/.gitignore @@ -120,4 +120,8 @@ venv.bak/ dmypy.json # Pyre type checker -.pyre/ \ No newline at end of file +.pyre/ + +## repository specific +/data/imagery +/data/solar.db diff --git a/README.md b/README.md index b3f31ec..da27dc0 100644 --- a/README.md +++ b/README.md @@ -8,23 +8,45 @@ I've chosen to use SQLite and SQLAlchemy for persisting the search locations and # Quickstart -To install requirements, run: +If you'd like to build and work out of the pre-existing Docker container, jump to the Docker container section just below. Regardless of the setup method, once you're all done you should be able to run: -`pip install -r requirements.txt` +`python run_entire_process.py --city --state ` -One of the requirements is rtree, which requires you also build libspatialindex, instructions [here](http://toblerity.org/rtree/install.html) (currently only required if you want to add filtering of existing OSM panels to your MapRoulette task to add the found panels to OSM) +And the whole suite of scripts should run, eventually outputting a MapRoulette challenge geoJSON for your city. (And leaving you with a sqlite database of these locations) -Also, currently [this DeepSolar repo](https://github.com/typicalTYLER/DeepSolar) must be present at ../DeepSolar (relative to this repo) and pre-trained weights must be present in the checkpoint directory. +Please create an [issue](https://github.com/typicalTYLER/SolarPanelDataWrangler/issues/new) if you have any trouble with this quickstart! -Lastly your Mapbox API key must be in your environment variables as MAPBOX_ACCESS_TOKEN="MY_ACCESS_TOKEN" +## Manual Setup -With all that taken care of, you can simply run: +### Conda Environment -`python run_entire_process.py --city --state ` +One of the requirements is rtree, which requires you to install libspatialindex, instructions [here](http://toblerity.org/rtree/install.html). -And the whole suite of scripts should run, eventually outputting a MapRoulette challenge geoJSON for your city. (And leaving you with a sqlite database of these locations) +To install the environment, use the `setup/environment_cpu.yml` file, e.g. -Please create an [issue](https://github.com/typicalTYLER/SolarPanelDataWrangler/issues/new) if you have any trouble with this quickstart! +``` +conda create --name spdw --file setup/environment_cpu.yml +``` + +*Note*: This was created using `conda=4.6.11` and `python=3.6.8`. + +### DeepSolar repository + +Currently, [this DeepSolar repo](https://github.com/typicalTYLER/DeepSolar) must be present at ../DeepSolar (relative to this repo) and pre-trained weights must be present in the `ckpt` directory inside of the `DeepSolar` repository. + +### MapBox Token + +Your Mapbox API key must be in your environment variables as MAPBOX_ACCESS_TOKEN="MY_ACCESS_TOKEN". + +## Docker Setup + +Within the `setup` directory is a `build_docker_images.sh` file that can be used to automatically setup a docker container that works out of the box with only a single additional step (adding your Mapbox API key). Once docker is set up, you simply need to specify a username that will be used inside the docker container, e.g. + +``` +bash setup/build_docker_images.sh --docker_user [specify username here] +``` + +After they're created, you should be able to work inside the container using the conda `spdw` environment that is set up, inside the the `repos` directory for the username you specified. # Overview @@ -45,4 +67,4 @@ maproulette.py contains functionality to turn positive classifications (above a # Contributing -Feel free to sign up for and submit pull requests for one of the [existing issues](https://github.com/typicalTYLER/SolarPanelDataWrangler/issues) if you want to contribute! I'm also down to add other Open Climate Fix collaborators as collaborators on this repo. Also feel free to create issues if you are having trouble with anything in this repo. \ No newline at end of file +Feel free to sign up for and submit pull requests for one of the [existing issues](https://github.com/typicalTYLER/SolarPanelDataWrangler/issues) if you want to contribute! I'm also down to add other Open Climate Fix collaborators as collaborators on this repo. Also feel free to create issues if you are having trouble with anything in this repo. diff --git a/setup/Dockerfile b/setup/Dockerfile new file mode 100644 index 0000000..2a25104 --- /dev/null +++ b/setup/Dockerfile @@ -0,0 +1,40 @@ +FROM spdw/base +LABEL maintainer="Sean Sall " + +ARG fname_environment_yml +ARG conda_version +ARG user +ARG branch + +USER $user + +RUN mkdir $HOME/repos && \ + cd $HOME/repos && \ + git clone https://github.com/typicalTYLER/SolarPanelDataWrangler.git && \ + git clone https://github.com/typicalTYLER/DeepSolar.git + +USER root +RUN cd /opt && \ + wget http://download.osgeo.org/libspatialindex/spatialindex-src-1.8.5.tar.gz && \ + tar -xvf spatialindex-src-1.8.5.tar.gz && \ + cd spatialindex-src-1.8.5 && \ + ./configure; make; make install + +USER $user +RUN cd $HOME/repos/SolarPanelDataWrangler &&\ + git checkout $branch && \ + cd $HOME/repos/SolarPanelDataWrangler/setup && \ + conda install conda=$conda_version && \ + conda env create -f $fname_environment_yml && \ + cd $HOME + +RUN mkdir -p ~/.config/matplotlib && \ + echo "backend: Agg" > ~/.config/matplotlib/matplotlibrc + +RUN cd $HOME/repos/DeepSolar && \ + mkdir ckpt && \ + cd ckpt && \ + wget https://s3-us-west-1.amazonaws.com/roofsolar/inception_classification.tar.gz && \ + wget https://s3-us-west-1.amazonaws.com/roofsolar/inception_segmentation.tar.gz && \ + tar xzf inception_classification.tar.gz && \ + tar xzf inception_segmentation.tar.gz diff --git a/setup/Dockerfile.base b/setup/Dockerfile.base new file mode 100644 index 0000000..21046fa --- /dev/null +++ b/setup/Dockerfile.base @@ -0,0 +1,42 @@ +ARG base_image=ubuntu:16.04 +FROM ${base_image} +LABEL maintainer="Sean Sall " + +ARG conda_version +ARG user + +ENV CONDA_DIRPATH /opt/conda +ENV PATH $CONDA_DIRPATH/bin:$PATH +ENV USER_UID 1000 + +RUN apt-get update && apt-get install -y \ + bzip2 \ + cmake \ + g++ \ + git \ + graphviz \ + libgl1-mesa-glx \ + libhdf5-dev \ + rtorrent \ + sudo \ + tmux \ + vim \ + wget + +RUN mkdir -p $CONDA_DIRPATH && \ + cd $CONDA_DIRPATH && \ + wget https://repo.continuum.io/miniconda/Miniconda3-${conda_version}-Linux-x86_64.sh && \ + chmod u+x Miniconda3-${conda_version}-Linux-x86_64.sh && \ + ./Miniconda3-${conda_version}-Linux-x86_64.sh -b -f -p $CONDA_DIRPATH && \ + conda install conda=4.6.11 && \ + conda install python=3.6.8 && \ + rm Miniconda3-${conda_version}-Linux-x86_64.sh + +RUN useradd -m -s /bin/bash -N -u $USER_UID $user && \ + echo "$user:$user" | chpasswd && adduser $user sudo && \ + chown -R $user $CONDA_DIRPATH && \ + echo "$user ALL=NOPASSWD: ALL" > /etc/sudoers.d/$user && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> /home/$user/.bashrc + +WORKDIR /home/$user +USER $user diff --git a/setup/build_docker_images.sh b/setup/build_docker_images.sh new file mode 100644 index 0000000..a72e36e --- /dev/null +++ b/setup/build_docker_images.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# build_docker_images.sh builds the necessary docker images for setting up the +# spdw environment and repository, using the Dockerfile.base and Dockerfile +# files in this directory. + +function usage { + echo "Usage: build_docker_image.sh [--branch] [--conda_version version] [--docker_user username]" + echo " [--fpath_custom_dockerfile fpath] [--gpu_build] " + echo " [--rebuild_image (base|final|custom)]" + echo "" + echo " --branch Branch to use when building, defaults to master." + echo " --conda_version Conda version for the Docker images - defaults " + echo " to 4.5.11, which is compatible with the repository " + echo " YMLs." + echo "" + echo " --docker_user Username for the Docker images. " + echo "" + echo " --fpath_custom_dockerfile fpath: Build a custom Docker image from a provided " + echo " Dockerfile after the Dockerfile.base and " + echo " Dockerfile image builds, meant to allow for " + echo " further customization of a user's environment. " + echo " This Dockerfile must use setup/Dockerfile as " + echo " the base image." + echo "" + echo " --gpu_build Build the base image (setup/Dockerfile.base) using " + echo " nvidia/cuda9.0-cudnn7-runtime-ubuntu16.04 as the " + echo " base image, instead of ubuntu:16.04 (the default)." + echo "" + echo " --rebuild_image Rebuild this image and any subsequent images that " + echo " use this one as base. Useful if you know that " + echo " something inside has changed (e.g. the conda " + echo " environment) but Docker won't be able to register " + echo " that change. Valid options are (base, final, custom)." + exit 1 +} + +while [ ! $# -eq 0 ] +do + case "$1" in + --branch) + BRANCH=$2 + shift 2 + ;; + --conda_version) + CONDA_VERSION=$2 + shift 2 + ;; + --docker_user) + DOCKER_USER=$2 + shift 2 + ;; + --fpath_custom_dockerfile) + FPATH_CUSTOM_DOCKERFILE=$2 + shift 2 + ;; + --gpu_build) + GPU_BUILD=true + shift 1 + ;; + --rebuild_image) + REBUILD_IMAGE=$2 + shift 2 + ;; + --help) + usage + shift + ;; + esac +done + +if [ -z "$DOCKER_USER" ]; then + echo "--docker_user flag must be specified!" + exit 1 +fi + +if [ -z "$BRANCH" ]; then + echo "--branch not specified, using the default of master." + BRANCH=master +fi + +if [ -z "$CONDA_VERSION" ]; then + echo "--conda_version not specified, using the default of 4.5.11." + CONDA_VERSION=4.5.11 +fi + +if [ ! -z "$REBUILD_IMAGE" ]; then + if ! [[ "$REBUILD_IMAGE" =~ ^(base|final|custom)$ ]]; then + echo "--rebuild_image option \"$REBUILD_IMAGE\" is not one of the " + echo "accepted options (base, final, or custom). If you'd like to " + echo "delete and remove one of the images, please specify one of " + echo "these options." + exit 1 + fi + + if [[ "$REBUILD_IMAGE" == "base" ]]; then + echo "--rebuild_image equal to \"base\"... will delete any existing, " + echo "spdw/base, spdw/final, and " + echo "spdw/custom images to build them anew." + DELETE_BASE=true + DELETE_FINAL=true + DELETE_CUSTOM=true + elif [[ "$REBUILD_IMAGE" == "final" ]]; then + echo "--rebuild_image equal to \"final\"... will delete any existing, " + echo "spdw/final and spdw/custom images to build " + echo "them anew." + DELETE_FINAL=true + DELETE_CUSTOM=true + elif [[ "$REBUILD_IMAGE" == "custom" ]]; then + echo "--rebuild_image equal to \"custom\"... will delete the " + echo "spdw/custom image to build it anew." + DELETE_CUSTOM=true + fi + + BASE_IMAGE_EXISTS=$(docker images -q spdw/base) + FINAL_IMAGE_EXISTS=$(docker images -q spdw/final) + CUSTOM_IMAGE_EXISTS=$(docker images -q spdw/custom) + + if [[ "$DELETE_BASE" == "true" ]] && [[ ! -z $BASE_IMAGE_EXISTS ]]; then + docker image rm spdw/base + fi + if [[ "$DELETE_FINAL" == "true" ]] && [[ ! -z $FINAL_IMAGE_EXISTS ]]; then + docker image rm spdw/final + fi + if [[ "$DELETE_CUSTOM" == "true" ]] && [[ ! -z $CUSTOM_IMAGE_EXISTS ]]; then + docker image rm spdw/custom + fi +fi + +BASE_IMAGE="ubuntu:16.04" +FNAME_ENVIRONMENT_YML="environment_cpu.yml" +if [[ ! -z "$GPU_BUILD" ]]; then + BASE_IMAGE="nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04" + FNAME_ENVIRONMENT_YML="environment_gpu.yml" +fi + +echo "Creating images with docker username $DOCKER_USER and miniconda " +echo "version $CONDA_VERSION..." + +docker build --build-arg user=$DOCKER_USER \ + --build-arg conda_version=$CONDA_VERSION \ + --build-arg base_image=$BASE_IMAGE \ + -t spdw/base --file ./Dockerfile.base ./ + +docker build --build-arg user=$DOCKER_USER \ + --build-arg branch=$BRANCH \ + --build-arg conda_version=$CONDA_VERSION \ + --build-arg fname_environment_yml=$FNAME_ENVIRONMENT_YML \ + -t spdw/final --file ./Dockerfile ./ + +if [ ! -z "$FPATH_CUSTOM_DOCKERFILE" ]; then + echo "Building custom Docker image based off of " + echo "$FPATH_CUSTOM_DOCKERFILE ..." + docker build --build-arg user=$DOCKER_USER \ + -t spdw/custom --file $FPATH_CUSTOM_DOCKERFILE ./ +fi diff --git a/setup/environment_cpu.yml b/setup/environment_cpu.yml new file mode 100644 index 0000000..563248f --- /dev/null +++ b/setup/environment_cpu.yml @@ -0,0 +1,174 @@ +name: spdw +channels: + - conda-forge + - defaults +dependencies: + - _tflow_select=2.3.0=mkl + - absl-py=0.7.0=py36_0 + - alembic=1.0.8=py_0 + - asn1crypto=0.24.0=py36_0 + - astor=0.7.1=py36_0 + - attrs=19.1.0=py36_1 + - backcall=0.1.0=py36_0 + - blas=1.0=mkl + - bzip2=1.0.6=h14c3975_5 + - c-ares=1.15.0=h7b6447c_1 + - ca-certificates=2019.1.23=0 + - cairo=1.14.12=h8948797_3 + - certifi=2019.3.9=py36_0 + - cffi=1.12.2=py36h2e261b9_1 + - chardet=3.0.4=py36_1 + - click=7.0=py36_0 + - click-plugins=1.0.4=py36_0 + - cligj=0.5.0=py36_0 + - cloudpickle=0.8.0=py36_0 + - cryptography=2.6.1=py36h1ba5d50_0 + - curl=7.64.0=hbc83047_2 + - cycler=0.10.0=py36_0 + - cytoolz=0.9.0.1=py36h14c3975_1 + - dask-core=1.1.4=py36_1 + - dbus=1.13.6=h746ee38_0 + - decorator=4.4.0=py36_1 + - descartes=1.1.0=py36_0 + - expat=2.2.6=he6710b0_0 + - fiona=1.8.4=py36hc38cc03_0 + - fontconfig=2.13.0=h9420a91_0 + - freetype=2.9.1=h8a8886c_1 + - freexl=1.0.5=h14c3975_0 + - gast=0.2.2=py36_0 + - gdal=2.3.3=py36hbb2a789_0 + - geojson=2.4.1=py_0 + - geojsonio=0.0.3=py_0 + - geopandas=0.4.1=py_0 + - geos=3.7.1=he6710b0_0 + - giflib=5.1.4=h14c3975_1 + - github3.py=1.3.0=py_0 + - glib=2.56.2=hd408876_0 + - grpcio=1.16.1=py36hf8bcb03_1 + - gst-plugins-base=1.14.0=hbbd80ab_1 + - gstreamer=1.14.0=hb453b48_1 + - h5py=2.9.0=py36h7918eee_0 + - hdf4=4.2.13=h3ca952b_2 + - hdf5=1.10.4=hb1b8bf9_0 + - icu=58.2=h9c2bf20_1 + - idna=2.8=py36_0 + - imageio=2.5.0=py36_0 + - intel-openmp=2019.3=199 + - ipython=7.4.0=py36h39e3cac_0 + - ipython_genutils=0.2.0=py36_0 + - jedi=0.13.3=py36_0 + - jpeg=9b=h024ee3a_2 + - json-c=0.13.1=h1bed415_0 + - jwcrypto=0.6.0=py_0 + - kealib=1.4.7=hd0c454d_6 + - keras-applications=1.0.7=py_0 + - keras-preprocessing=1.0.9=py_0 + - kiwisolver=1.0.1=py36hf484d3e_0 + - krb5=1.16.1=h173b8e3_7 + - libboost=1.67.0=h46d08c1_4 + - libcurl=7.64.0=h20c2e04_2 + - libdap4=3.19.1=h6ec2957_0 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=8.2.0=hdf63c60_1 + - libgdal=2.3.3=h2e7e64b_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libkml=1.3.0=h590aaf7_4 + - libnetcdf=4.6.1=h11d0813_2 + - libpng=1.6.36=hbc83047_0 + - libpq=11.2=h20c2e04_0 + - libprotobuf=3.6.1=hd408876_0 + - libspatialindex=1.8.5=h20b78c2_2 + - libspatialite=4.3.0a=hb08deb6_19 + - libssh2=1.8.0=h1ba5d50_4 + - libstdcxx-ng=8.2.0=hdf63c60_1 + - libtiff=4.0.10=h2733197_2 + - libuuid=1.0.3=h1bed415_2 + - libxcb=1.13=h1bed415_1 + - libxml2=2.9.9=he19cac6_0 + - mapclassify=2.0.1=py_0 + - markdown=3.0.1=py36_0 + - markupsafe=1.1.1=py36h7b6447c_0 + - matplotlib=3.0.3=py36h5429711_0 + - mkl=2019.3=199 + - mkl_fft=1.0.10=py36ha843d7b_0 + - mkl_random=1.0.2=py36hd81dba3_0 + - munch=2.3.2=py36_0 + - ncurses=6.1=he6710b0_1 + - ndg-httpsclient=0.5.1=py_1 + - networkx=2.2=py36_1 + - numpy=1.16.2=py36h7e9f1db_0 + - numpy-base=1.16.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openjpeg=2.3.0=h05c96fa_1 + - openssl=1.1.1b=h7b6447c_1 + - overpy=0.4=py_0 + - pandas=0.24.2=py36he6710b0_0 + - parso=0.3.4=py36_0 + - pcre=8.43=he6710b0_0 + - pexpect=4.6.0=py36_0 + - pickleshare=0.7.5=py36_0 + - pillow=5.4.1=py36h34e0f95_0 + - pip=19.0.3=py36_0 + - pixman=0.38.0=h7b6447c_0 + - poppler=0.65.0=h581218d_1 + - poppler-data=0.4.9=0 + - proj4=5.2.0=he6710b0_1 + - prompt_toolkit=2.0.9=py36_0 + - protobuf=3.6.1=py36he6710b0_0 + - psycopg2=2.7.6.1=py36h1ba5d50_0 + - ptyprocess=0.6.0=py36_0 + - pyasn1=0.4.4=py_1 + - pycparser=2.19=py36_0 + - pygments=2.3.1=py36_0 + - pyopenssl=19.0.0=py36_0 + - pyparsing=2.3.1=py36_0 + - pyproj=1.9.6=py36h14380d9_0 + - pyqt=5.9.2=py36h05f1152_2 + - pysocks=1.6.8=py36_0 + - python=3.6.8=h0371630_0 + - python-dateutil=2.8.0=py36_0 + - python-editor=1.0.4=py_0 + - pytz=2018.9=py36_0 + - pywavelets=1.0.2=py36hdd07704_0 + - qt=5.9.7=h5867ecd_1 + - readline=7.0=h7b6447c_5 + - requests=2.21.0=py36_0 + - rtree=0.8.3=py36_0 + - scikit-image=0.14.2=py36he6710b0_0 + - scipy=1.2.1=py36h7c811a0_0 + - setuptools=40.8.0=py36_0 + - shapely=1.6.4=py36h86c5351_0 + - sip=4.19.8=py36hf484d3e_0 + - six=1.12.0=py36_0 + - sqlalchemy=1.3.1=py36h7b6447c_0 + - sqlite=3.27.2=h7b6447c_0 + - tensorboard=1.12.2=py36he6710b0_0 + - tensorflow=1.12.0=mkl_py36h69b6ba0_0 + - tensorflow-base=1.12.0=mkl_py36h3c3e929_0 + - termcolor=1.1.0=py36_1 + - tk=8.6.8=hbc83047_0 + - toolz=0.9.0=py36_0 + - tornado=6.0.2=py36h7b6447c_0 + - traitlets=4.3.2=py36_0 + - uritemplate.py=3.0.2=py_1 + - urllib3=1.24.1=py36_0 + - wcwidth=0.1.7=py36_0 + - werkzeug=0.14.1=py36_0 + - wheel=0.33.1=py36_0 + - xerces-c=3.2.2=h780794e_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - boto3==1.9.130 + - botocore==1.12.130 + - cachecontrol==0.12.5 + - docutils==0.14 + - iso3166==1.0 + - jmespath==0.9.4 + - mako==1.0.7 + - mapbox==0.18.0 + - msgpack==0.6.1 + - polyline==1.3.2 + - s3transfer==0.2.0 From 21830c2ed3aec3f34e8237b492768b61dd94e6a1 Mon Sep 17 00:00:00 2001 From: Tyler Busby Date: Fri, 12 Apr 2019 18:24:00 -0500 Subject: [PATCH 2/6] Change geojsonio preview to be a printed link, for ease of use in a docker container. --- run_entire_process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_entire_process.py b/run_entire_process.py index 5abc776..391bae4 100644 --- a/run_entire_process.py +++ b/run_entire_process.py @@ -50,9 +50,9 @@ polygon = process_city_shapes.simplify_polygon(polygon) if not args.no_geojsonio: - # Open the simplified polygon in a web window to double check correctness - geojsonio.display(geopandas.GeoSeries([polygon]).to_json()) - input("A geojson.io window has been opened with your simplified search polygon, press enter to continue if it " + # Create a link to geojsonio for the polygon to double check correctness + print(geojsonio.make_url(geopandas.GeoSeries([polygon]).to_json())) + input("A geojson.io link has been created with your simplified search polygon, press enter to continue if it " "looks okay. If it doesn't, implement a way to edit your polygon and feed it directly to this script :)") print("Calculating the coordinates of the imagery grid contained within this polygon.") From 46c16a1ec7f4588b7162c837b586f15d35b58fb0 Mon Sep 17 00:00:00 2001 From: Tyler Busby Date: Fri, 12 Apr 2019 18:24:54 -0500 Subject: [PATCH 3/6] Refactoring, add helper function to get big clusters --- gather_city_shapes.py | 12 ++++++------ solardb.py | 27 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/gather_city_shapes.py b/gather_city_shapes.py index f7d47d9..cb4f1ba 100644 --- a/gather_city_shapes.py +++ b/gather_city_shapes.py @@ -1,9 +1,9 @@ -import math -import os +import argparse +import csv import json +import os + import requests -import csv -import argparse def get_filename(city, state): @@ -27,8 +27,8 @@ def get_city_state_filepaths(csvpath): def gather(csvpath): for city, state, filepath in get_city_state_filepaths(csvpath): if not os.path.isfile(filepath): - with open(filepath, 'w') as outfile: - json.dump(query_nominatim_for_geojson(city, state), outfile) + with open(filepath, 'w') as outfile: + json.dump(query_nominatim_for_geojson(city, state), outfile) def query_nominatim_for_geojson(city=None, state=None, county=None, country=None): diff --git a/solardb.py b/solardb.py index c14fa8a..22cedef 100644 --- a/solardb.py +++ b/solardb.py @@ -3,11 +3,14 @@ import math import overpy -from sqlalchemy import Column, Integer, String, ForeignKey, Float, Boolean, PrimaryKeyConstraint, Index, desc +from sqlalchemy import Column, Integer, String, ForeignKey, Float, Boolean, PrimaryKeyConstraint, Index, desc, alias from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker, relationship from sqlalchemy.sql import expression +from sqlalchemy.sql.functions import count + +from process_city_shapes import num2deg Base = declarative_base() # TODO improve session management @@ -257,3 +260,25 @@ def get_osm_pv_nodes(): nodes = session.query(OSMSolarNode).all() session.close() return [(node.longitude, node.latitude) for node in nodes] + + +def get_lat_lon_for_largest_clusters(limit=10, polygon_name=None): + """ + Queries the db for largest contiguous clusters + + :param limit: number of results to return + :param polygon_name: optional name to filter for + :return: list of lat_lon tuples near each cluster (exact center doesn't really matter for my use case) + """ + session = Session() + cluster_query = session.query(SlippyTile.cluster_id).filter(SlippyTile.cluster_id.isnot(None)) + if polygon_name: + cluster_query = cluster_query.filter(SlippyTile.polygon_name == polygon_name) + tuple_list = cluster_query.group_by(SlippyTile.cluster_id).order_by(desc(count(SlippyTile.cluster_id))).limit( + limit).all() + lat_lons = [] + for cluster_id, in tuple_list: + lat_lons.append(reversed(num2deg(session.query(SlippyTile.column, SlippyTile.row).filter( + SlippyTile.cluster_id == cluster_id).limit(1).first()))) + session.close() + return lat_lons From 845cc2523014844b7e2c75618a6d4da6013cb1b0 Mon Sep 17 00:00:00 2001 From: Tyler Busby Date: Fri, 12 Apr 2019 18:27:14 -0500 Subject: [PATCH 4/6] Revert "Refactoring, add helper function to get big clusters" This reverts commit 46c16a1ec7f4588b7162c837b586f15d35b58fb0. --- gather_city_shapes.py | 12 ++++++------ solardb.py | 27 +-------------------------- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/gather_city_shapes.py b/gather_city_shapes.py index cb4f1ba..f7d47d9 100644 --- a/gather_city_shapes.py +++ b/gather_city_shapes.py @@ -1,9 +1,9 @@ -import argparse -import csv -import json +import math import os - +import json import requests +import csv +import argparse def get_filename(city, state): @@ -27,8 +27,8 @@ def get_city_state_filepaths(csvpath): def gather(csvpath): for city, state, filepath in get_city_state_filepaths(csvpath): if not os.path.isfile(filepath): - with open(filepath, 'w') as outfile: - json.dump(query_nominatim_for_geojson(city, state), outfile) + with open(filepath, 'w') as outfile: + json.dump(query_nominatim_for_geojson(city, state), outfile) def query_nominatim_for_geojson(city=None, state=None, county=None, country=None): diff --git a/solardb.py b/solardb.py index 22cedef..c14fa8a 100644 --- a/solardb.py +++ b/solardb.py @@ -3,14 +3,11 @@ import math import overpy -from sqlalchemy import Column, Integer, String, ForeignKey, Float, Boolean, PrimaryKeyConstraint, Index, desc, alias +from sqlalchemy import Column, Integer, String, ForeignKey, Float, Boolean, PrimaryKeyConstraint, Index, desc from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker, relationship from sqlalchemy.sql import expression -from sqlalchemy.sql.functions import count - -from process_city_shapes import num2deg Base = declarative_base() # TODO improve session management @@ -260,25 +257,3 @@ def get_osm_pv_nodes(): nodes = session.query(OSMSolarNode).all() session.close() return [(node.longitude, node.latitude) for node in nodes] - - -def get_lat_lon_for_largest_clusters(limit=10, polygon_name=None): - """ - Queries the db for largest contiguous clusters - - :param limit: number of results to return - :param polygon_name: optional name to filter for - :return: list of lat_lon tuples near each cluster (exact center doesn't really matter for my use case) - """ - session = Session() - cluster_query = session.query(SlippyTile.cluster_id).filter(SlippyTile.cluster_id.isnot(None)) - if polygon_name: - cluster_query = cluster_query.filter(SlippyTile.polygon_name == polygon_name) - tuple_list = cluster_query.group_by(SlippyTile.cluster_id).order_by(desc(count(SlippyTile.cluster_id))).limit( - limit).all() - lat_lons = [] - for cluster_id, in tuple_list: - lat_lons.append(reversed(num2deg(session.query(SlippyTile.column, SlippyTile.row).filter( - SlippyTile.cluster_id == cluster_id).limit(1).first()))) - session.close() - return lat_lons From 36aff2cc5ef702148b1ea2e9d9abf2f3086b70d0 Mon Sep 17 00:00:00 2001 From: Sean Sall Date: Sat, 13 Apr 2019 02:04:39 +0000 Subject: [PATCH 5/6] Add environment_gpu.yml --- setup/environment_gpu.yml | 175 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 setup/environment_gpu.yml diff --git a/setup/environment_gpu.yml b/setup/environment_gpu.yml new file mode 100644 index 0000000..30b8fa6 --- /dev/null +++ b/setup/environment_gpu.yml @@ -0,0 +1,175 @@ +name: spdw +channels: + - conda-forge + - defaults +dependencies: + - _tflow_select==2.1.0=gpu + - absl-py=0.7.0=py36_0 + - alembic=1.0.8=py_0 + - asn1crypto=0.24.0=py36_0 + - astor=0.7.1=py36_0 + - attrs=19.1.0=py36_1 + - backcall=0.1.0=py36_0 + - blas=1.0=mkl + - bzip2=1.0.6=h14c3975_5 + - c-ares=1.15.0=h7b6447c_1 + - ca-certificates=2019.1.23=0 + - cairo=1.14.12=h8948797_3 + - certifi=2019.3.9=py36_0 + - cffi=1.12.2=py36h2e261b9_1 + - chardet=3.0.4=py36_1 + - click=7.0=py36_0 + - click-plugins=1.0.4=py36_0 + - cligj=0.5.0=py36_0 + - cloudpickle=0.8.0=py36_0 + - cryptography=2.6.1=py36h1ba5d50_0 + - curl=7.64.0=hbc83047_2 + - cycler=0.10.0=py36_0 + - cytoolz=0.9.0.1=py36h14c3975_1 + - dask-core=1.1.4=py36_1 + - dbus=1.13.6=h746ee38_0 + - decorator=4.4.0=py36_1 + - descartes=1.1.0=py36_0 + - expat=2.2.6=he6710b0_0 + - fiona=1.8.4=py36hc38cc03_0 + - fontconfig=2.13.0=h9420a91_0 + - freetype=2.9.1=h8a8886c_1 + - freexl=1.0.5=h14c3975_0 + - gast=0.2.2=py36_0 + - gdal=2.3.3=py36hbb2a789_0 + - geojson=2.4.1=py_0 + - geojsonio=0.0.3=py_0 + - geopandas=0.4.1=py_0 + - geos=3.7.1=he6710b0_0 + - giflib=5.1.4=h14c3975_1 + - github3.py=1.3.0=py_0 + - glib=2.56.2=hd408876_0 + - grpcio=1.16.1=py36hf8bcb03_1 + - gst-plugins-base=1.14.0=hbbd80ab_1 + - gstreamer=1.14.0=hb453b48_1 + - h5py=2.9.0=py36h7918eee_0 + - hdf4=4.2.13=h3ca952b_2 + - hdf5=1.10.4=hb1b8bf9_0 + - icu=58.2=h9c2bf20_1 + - idna=2.8=py36_0 + - imageio=2.5.0=py36_0 + - intel-openmp=2019.3=199 + - ipython=7.4.0=py36h39e3cac_0 + - ipython_genutils=0.2.0=py36_0 + - jedi=0.13.3=py36_0 + - jpeg=9b=h024ee3a_2 + - json-c=0.13.1=h1bed415_0 + - jwcrypto=0.6.0=py_0 + - kealib=1.4.7=hd0c454d_6 + - keras-applications=1.0.7=py_0 + - keras-preprocessing=1.0.9=py_0 + - kiwisolver=1.0.1=py36hf484d3e_0 + - krb5=1.16.1=h173b8e3_7 + - libboost=1.67.0=h46d08c1_4 + - libcurl=7.64.0=h20c2e04_2 + - libdap4=3.19.1=h6ec2957_0 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=8.2.0=hdf63c60_1 + - libgdal=2.3.3=h2e7e64b_0 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libkml=1.3.0=h590aaf7_4 + - libnetcdf=4.6.1=h11d0813_2 + - libpng=1.6.36=hbc83047_0 + - libpq=11.2=h20c2e04_0 + - libprotobuf=3.6.1=hd408876_0 + - libspatialindex=1.8.5=h20b78c2_2 + - libspatialite=4.3.0a=hb08deb6_19 + - libssh2=1.8.0=h1ba5d50_4 + - libstdcxx-ng=8.2.0=hdf63c60_1 + - libtiff=4.0.10=h2733197_2 + - libuuid=1.0.3=h1bed415_2 + - libxcb=1.13=h1bed415_1 + - libxml2=2.9.9=he19cac6_0 + - mapclassify=2.0.1=py_0 + - markdown=3.0.1=py36_0 + - markupsafe=1.1.1=py36h7b6447c_0 + - matplotlib=3.0.3=py36h5429711_0 + - mkl=2019.3=199 + - mkl_fft=1.0.10=py36ha843d7b_0 + - mkl_random=1.0.2=py36hd81dba3_0 + - munch=2.3.2=py36_0 + - ncurses=6.1=he6710b0_1 + - ndg-httpsclient=0.5.1=py_1 + - networkx=2.2=py36_1 + - numpy=1.16.2=py36h7e9f1db_0 + - numpy-base=1.16.2=py36hde5b4d6_0 + - olefile=0.46=py36_0 + - openjpeg=2.3.0=h05c96fa_1 + - openssl=1.1.1b=h7b6447c_1 + - overpy=0.4=py_0 + - pandas=0.24.2=py36he6710b0_0 + - parso=0.3.4=py36_0 + - pcre=8.43=he6710b0_0 + - pexpect=4.6.0=py36_0 + - pickleshare=0.7.5=py36_0 + - pillow=5.4.1=py36h34e0f95_0 + - pip=19.0.3=py36_0 + - pixman=0.38.0=h7b6447c_0 + - poppler=0.65.0=h581218d_1 + - poppler-data=0.4.9=0 + - proj4=5.2.0=he6710b0_1 + - prompt_toolkit=2.0.9=py36_0 + - protobuf=3.6.1=py36he6710b0_0 + - psycopg2=2.7.6.1=py36h1ba5d50_0 + - ptyprocess=0.6.0=py36_0 + - pyasn1=0.4.4=py_1 + - pycparser=2.19=py36_0 + - pygments=2.3.1=py36_0 + - pyopenssl=19.0.0=py36_0 + - pyparsing=2.3.1=py36_0 + - pyproj=1.9.6=py36h14380d9_0 + - pyqt=5.9.2=py36h05f1152_2 + - pysocks=1.6.8=py36_0 + - python=3.6.8=h0371630_0 + - python-dateutil=2.8.0=py36_0 + - python-editor=1.0.4=py_0 + - pytz=2018.9=py36_0 + - pywavelets=1.0.2=py36hdd07704_0 + - qt=5.9.7=h5867ecd_1 + - readline=7.0=h7b6447c_5 + - requests=2.21.0=py36_0 + - rtree=0.8.3=py36_0 + - scikit-image=0.14.2=py36he6710b0_0 + - scipy=1.2.1=py36h7c811a0_0 + - setuptools=40.8.0=py36_0 + - shapely=1.6.4=py36h86c5351_0 + - sip=4.19.8=py36hf484d3e_0 + - six=1.12.0=py36_0 + - sqlalchemy=1.3.1=py36h7b6447c_0 + - sqlite=3.27.2=h7b6447c_0 + - tensorboard=1.12.2=py36he6710b0_0 + - tensorflow=1.12.0=gpu_py36he74679b_0 + - tensorflow-base=1.12.0=gpu_py36had579c0_0 + - tensorflow-gpu=1.12.0=h0d30ee6_0 + - termcolor=1.1.0=py36_1 + - tk=8.6.8=hbc83047_0 + - toolz=0.9.0=py36_0 + - tornado=6.0.2=py36h7b6447c_0 + - traitlets=4.3.2=py36_0 + - uritemplate.py=3.0.2=py_1 + - urllib3=1.24.1=py36_0 + - wcwidth=0.1.7=py36_0 + - werkzeug=0.14.1=py36_0 + - wheel=0.33.1=py36_0 + - xerces-c=3.2.2=h780794e_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.3.7=h0b5b093_0 + - pip: + - boto3==1.9.130 + - botocore==1.12.130 + - cachecontrol==0.12.5 + - docutils==0.14 + - iso3166==1.0 + - jmespath==0.9.4 + - mako==1.0.7 + - mapbox==0.18.0 + - msgpack==0.6.1 + - polyline==1.3.2 + - s3transfer==0.2.0 From e6b720486d6c901518e3c4de6ad3cdf00e9e906f Mon Sep 17 00:00:00 2001 From: Sean Sall Date: Mon, 15 Apr 2019 15:31:19 +0000 Subject: [PATCH 6/6] Update README --- README.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index da27dc0..109d7df 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,13 @@ Please create an [issue](https://github.com/typicalTYLER/SolarPanelDataWrangler/ One of the requirements is rtree, which requires you to install libspatialindex, instructions [here](http://toblerity.org/rtree/install.html). -To install the environment, use the `setup/environment_cpu.yml` file, e.g. +To install the environment, choose either the `setup/environment_cpu.yml` and `setup/environment_gpu.yml`. If you have a GPU that you intend to use, you'll need to setup the relevant packages / drivers (e.g. cuDNN / CUDA, NVIDIA diver, etc.) before installing the conda environment. Once you've chosen a YML file, you can create the environment via something like the following: ``` conda create --name spdw --file setup/environment_cpu.yml ``` -*Note*: This was created using `conda=4.6.11` and `python=3.6.8`. +*Note*: These environments are built on `conda=4.6.11`, `python=3.6.8`, and `tensorflow=1.12.0`. ### DeepSolar repository @@ -40,13 +40,27 @@ Your Mapbox API key must be in your environment variables as MAPBOX_ACCESS_TOKEN ## Docker Setup -Within the `setup` directory is a `build_docker_images.sh` file that can be used to automatically setup a docker container that works out of the box with only a single additional step (adding your Mapbox API key). Once docker is set up, you simply need to specify a username that will be used inside the docker container, e.g. +Within the `setup` directory is a `build_docker_images.sh` script that can be used to automatically setup a docker container to develop out of. + +### No GPU Build + +To build a docker image that uses the no GPU conda environment, first install docker. Next, choose a username that you would like to use inside the container, and from **within the setup directory**, run: + +``` +bash build_docker_images.sh --docker_user [specify username here] +``` + +Once the image is built, you should be able to work inside a container created from the image just as if you were logged into a remote instance. + +### GPU Build + +To build a docker image that uses the GPU conda environment, first install nvidia-docker (version 2.0). Next, make sure that you have the appropriate GPU driver installed for your system and for the version of tensorflow that will be used (`1.12.0`) as well as the versions of CUDA and cuDNN (CUDA 9.0 and cuDNN7. Once that is done, choose a username that you would like to use inside the container, and from **within the setup directory**, run: ``` -bash setup/build_docker_images.sh --docker_user [specify username here] +bash build_docker_images.sh --docker_user [specify username here] --gpu_build ``` -After they're created, you should be able to work inside the container using the conda `spdw` environment that is set up, inside the the `repos` directory for the username you specified. +Once the image is built, you should be able to work inside a container created from the image just as if you were logged into a remote instance. # Overview