# Import data handling libraries
import json
import os
import pickle
import numpy as np
import pandas as pd
import dask as da
from multiprocessing import Pool

# graphical control libraries
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
import geopandas as gpd
import fiona
import shapely.ops
import ftplib
import urllib as urllib2
import wget
import bz2

# shape and layer libraries
from descartes import PolygonPatch
from shapely.geometry import MultiPolygon, Polygon, box, point, shape
from matplotlib.collections import PatchCollection
from mpl_toolkits.basemap import Basemap
from bs4 import BeautifulSoup as bs

def saveDictOfDf(outfilepath, dictionaryObject):
    """
    Save a json file from a pickle'd python dictionary-of-dataframes object

    outfilepath: (dir) the path to the output json file
    dictionaryObject: (dict) the python dictionary object
    """
    # write a dictionary of dataframes to a json file using pickle
    with open(outfilepath, 'wb') as f:
        pickle.dump(dictionaryObject, f)
        f.close()

def readDictOfDf(infilepath):
    """
    Read in a json file that contains pickle'd python objects

    infilepath: (dir) the path to the input json file
    """
    # read a dictionary of dataframes from a json file using pickle
    with open(infilepath, 'rb') as f:
        dictionaryObject = pickle.load(f)
        f.close()
    return(dictionaryObject)

def reprojShapefile(sourcepath, outpath=None, newprojdictionary={'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84'}):
    """
    Convert a shapefile into a new projection

    sourcepath: (dir) the path to the .shp file
    newprojdictionary: (dict) the new projection definitions (default is longlat projection with WGS84 datum)
    outpath: (dir) the output path for the new shapefile
    """
    # if outpath is none, treat the reprojection as a file replacement
    if isinstance(outpath, type(None)):
        outpath = sourcepath

    shpfile = gpd.GeoDataFrame.from_file(sourcepath)
    shpfile = shpfile.to_crs(newprojdictionary)
    shpfile.to_file(outpath)

def getFullShape(shapefile):
    """
    Generate a MultiPolygon to represent each shape/polygon within the shapefile

    shapefile: (dir) a path to the ESRI .shp shapefile
    """
    shp = fiona.open(shapefile)
    mp = [shape(pol['geometry']) for pol in shp]
    mp = MultiPolygon(mp)
    shp.close()
    return(mp)

def getShapeBbox(polygon):
    """
    Generate a geometric box to represent the bounding box for the polygon, shapefile connection, or MultiPolygon

    polygon: (geometry) a geometric polygon, MultiPolygon, or shapefile connection
    """
    # identify the cardinal bounds
    minx, miny, maxx, maxy = polygon.bounds
    bbox = box(minx, miny, maxx, maxy, ccw=True)
    return(bbox)

def readShapefileTable(shapefile):
    """
    Read in the datatable captured within the shapefile properties

    shapefile: (dir) a path to the ESRI .shp shapefile
    """
    shp = fiona.open(shapefile)
    centroid = [eachpol['properties'] for eachpol in shp]
    cent_df = pd.DataFrame.from_dict(centroid, orient='columns')
    shp.close()
    return(cent_df)

def filterPointsinShape(shape, points_lat, points_lon, points_elev=None, buffer_distance=0.06,
                        buffer_resolution=16, labels=['LAT', 'LONG_', 'ELEV']):
    """
    Filter for datafiles that can be used

    shape: (geometry) a geometric polygon or MultiPolygon
    points_lat: (series) a series of latitude points in WGS84 projection
    points_lon: (series) a series of longitude points in WGS84 projection
    points_elev: (series) a series of elevation points in meters; default is None
    buffer_distance: (float64) a numerical multiplier to increase the geodetic boundary area
    buffer_resolution: (float64) the increments between geodetic longlat degrees
    labels: (list) a list of preferred labels for latitude, longitude, and elevation
    """
    # add buffer region
    region = shape.buffer(buffer_distance, resolution=buffer_resolution)

    # construct bounds
    minx, miny, maxx, maxy = region.bounds

    # construct points_elev if null
    if isinstance(points_elev, type(None)):
        points_elev = np.repeat(np.nan, len(points_lon))

    # filter the points to the regional bounds
    bound_filter = points_lat.map(lambda y: (y >= miny) and (y <= maxy)) & points_lon.map(lambda x: (x >= minx) and (x <= maxx))

    # Intersection each coordinate with the region
    limited_list = []
    for lon, lat, elev in zip(points_lon[bound_filter], points_lat[bound_filter], points_elev[bound_filter]):
        gpoint = point.Point(lon, lat)
        if gpoint.intersects(region):
            limited_list.append([lat, lon, elev])

    maptable = pd.DataFrame.from_records(limited_list, columns=labels)
    return(maptable)

def scrapeurl(url, startswith=None, hasKeyword=None):
    """
    Scrape hyperlink references from in a url

    url: (str) the web folder path to be scraped for hyperlink references
    startswith: (str) the starting keywords for a webpage element; default is None
    hasKeyword: (str) keywords represented in a webpage element; default is None
    """
    # grab the html of the url, and prettify the html structure
    page = urllib2.urlopen(url).read()
    page_soup = bs(page, 'lxml')
    page_soup.prettify()

    # loop through and filter the hyperlinked lines
    if pd.isnull(startswith):
        temp = [anchor['href'] for anchor in page_soup.findAll('a', href=True) if hasKeyword in anchor['href']]
    else:
        temp = [anchor['href'] for anchor in page_soup.findAll('a', href=True) if anchor['href'].startswith(startswith)]

    # convert to dataframe then separate the lon and lat as float coordinate values
    temp = pd.DataFrame(temp, columns=['filenames'])
    return(temp)

def treatgeoself(shapefile, NAmer, mappingfile=os.path.join(os.getcwd(), 'mappingfile.csv'), buffer_distance=0.06):
    """
    TreatGeoSelf to some [data] lovin'!

    shapefile: (dir) the path to an ESRI shapefile for the region of interest
    Namer: (dir) the path to the ESRI shapefile, which has each 1/16th-degree gridded cell centroid and DEM elevation
    mappingfile: (str) the name of the output file; default is 'mappingfile.csv'
    buffer_distance: (float64) the multiplier for increasing the geodetic boundary area; default is 0.06
    """
    # conform projections to longlat values in WGS84
    reprojShapefile(shapefile, newprojdictionary={'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84'}, outpath=None)

    # read shapefile into a multipolygon shape-object
    shape_mp = getFullShape(shapefile)

    # read in the North American continental DEM points for the station elevations
    NAmer_datapoints = readShapefileTable(NAmer).rename(columns={'Lat': 'LAT', 'Long': 'LONG_', 'Elev': 'ELEV'})

    # generate maptable
    maptable = filterPointsinShape(shape_mp,
                                   points_lat=NAmer_datapoints.LAT,
                                   points_lon=NAmer_datapoints.LONG_,
                                   points_elev=NAmer_datapoints.ELEV,
                                   buffer_distance=buffer_distance, buffer_resolution=16,
                                   labels=['LAT', 'LONG_', 'ELEV'])
    maptable.reset_index(inplace=True)
    maptable = maptable.rename(columns={'index': 'FID'})
    print(maptable.shape)
    print(maptable.head())

    # print the mappingfile
    maptable.to_csv(mappingfile, sep=',', header=True, index=False)
    return(mappingfile)

def mapContentFolder(resid):
    """
    Map the content folder path for a hydroshare resource migrated to HydroShare JupyterHub

    resid: (str) a string hash that represents the hydroshare resource that has been migrated
    """
    path = os.path.join('/home/jovyan/work/notebooks/data', str(resid), str(resid), 'data/contents')
    return(path)

def canadabox_bc():
    """
    Establish the Canadian Columbia river basin bounding box
    """
    # left, bottom, right top, ccw=True
    return(box(-138.0, 49.0, -114.0, 53.0))

def scrape_domain(domain, subdomain, startswith=None):
    """
    Scrape an ftp domain for the hyperlink references to subfolders

    domain: (str) the web folder path
    subdomain: (str) the subfolder path to be scraped for hyperlink references
    startswith: (str) the starting keywords for a webpage element; default is None
    """
    # connect to domain
    ftp = ftplib.FTP(domain)
    ftp.login()
    ftp.cwd(subdomain)

    # scrape for data directories
    tmp = [dirname for dirname in ftp.nlst() if dirname.startswith(startswith)]
    geodf = pd.DataFrame(tmp, columns=['dirname'])

    # conform to bounding box format
    tmp = geodf['dirname'].apply(lambda x: x.split('.')[1:])
    tmp = tmp.apply(lambda x: list(map(float, x)) if len(x) > 2 else x)

    # assemble the boxes
    geodf['bbox'] = tmp.apply(lambda x: box(x[0]*-1, x[2]-1, x[1]*-1, x[3]) if len(x) > 2 else canadabox_bc())
    return(geodf)

def mapToBlock(df_points, df_regions):
    """
    Map block membership for each coordinate point

    df_points: (dataframe) a dataframe containing the lat and long for each time-series datafile
    dr_regions: (dataframe) a dataframe containing the bounding box (bbox) for each block cluster
    """
    for index, eachblock in df_regions.iterrows():
        for ind, row in df_points.iterrows():
            if point.Point(row['LONG_'], row['LAT']).intersects(eachblock['bbox']):
                df_points.loc[ind, 'blocks'] = str(eachblock['dirname'])
    return(df_points)

# ### CIG (DHSVM)-oriented functions

def compile_bc_Livneh2013_locations(maptable):
    """
    Compile a list of file URLs for bias corrected Livneh et al. 2013 (CIG)

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    locations = []
    for ind, row in maptable.iterrows():
        basename = '_'.join(['data', str(row['LAT']), str(row['LONG_'])])
        url = ['http://cses.washington.edu/rocinante/Livneh/bcLivneh_WWA_2013/forcings_ascii/', basename]
        locations.append(''.join(url))
    return(locations)

def compile_Livneh2013_locations(maptable):
    """
    Compile a list of file URLs for Livneh et al. 2013 (CIG)

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    locations = []
    for ind, row in maptable.iterrows():
        basename = '_'.join(['data', str(row['LAT']), str(row['LONG_'])])
        url = ['http://www.cses.washington.edu/rocinante/Livneh/Livneh_WWA_2013/forcs_dhsvm/', basename]
        locations.append(''.join(url))
    return(locations)

# VIC-oriented functions

def compile_VICASCII_Livneh2013_locations(maptable):
    """
    Compile the list of file URLs for Livneh et al., 2013 VIC.ASCII outputs

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    # gridded data product metadata
    domain = 'livnehpublicstorage.colorado.edu'
    subdomain = 'public/Livneh.2013.CONUS.Dataset/Fluxes.asc.v.1.2.1915.2011.bz2'

    # identify the subfolder blocks
    blocks = scrape_domain(domain=domain, subdomain=subdomain, startswith='fluxes')

    # map each coordinate to the subfolder
    maptable = mapToBlock(maptable, blocks)

    locations = []
    for ind, row in maptable.iterrows():
        loci = '_'.join(['VIC_fluxes_Livneh_CONUSExt_v.1.2_2013', str(row['LAT']), str(row['LONG_'])])
        url = os.path.join('ftp://'+domain, subdomain, str(row['blocks']), loci+'.bz2')
        locations.append(url)
    return(locations)

def compile_VICASCII_Livneh2015_locations(maptable):
    """
    Compile the list of file URLs for Livneh et al., 2015 VIC.ASCII outputs

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    # gridded data product metadata
    domain = '192.12.137.7'
    subdomain = 'pub/dcp/archive/OBS/livneh2014.1_16deg/VIC.ASCII'

    locations = []
    for ind, row in maptable.iterrows():
        loci = '_'.join(['Fluxes_Livneh_NAmerExt_15Oct2014', str(row['LAT']), str(row['LONG_'])])
        url = os.path.join('ftp://'+domain, subdomain, 'latitude.'+str(row['LAT']), loci+'.bz2')
        locations.append(url)
    return(locations)

# Climate (Meteorological observations)-oriented functions

def compile_dailyMET_Livneh2013_locations(maptable):
    """
    Compile the list of file URLs for Livneh et al., 2013 Daily Meteorology data

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    # gridded data product metadata
    domain = 'livnehpublicstorage.colorado.edu'
    subdomain = 'public/Livneh.2013.CONUS.Dataset/Meteorology.asc.v.1.2.1915.2011.bz2'

    # identify the subfolder blocks
    blocks = scrape_domain(domain=domain, subdomain=subdomain, startswith='data')

    # map each coordinate to the subfolder
    maptable = mapToBlock(maptable, blocks)

    locations = []
    for ind, row in maptable.iterrows():
        loci = '_'.join(['Meteorology_Livneh_CONUSExt_v.1.2_2013', str(row['LAT']), str(row['LONG_'])])
        url = os.path.join('ftp://'+domain, subdomain, str(row['blocks']), loci+'.bz2')
        locations.append(url)
    return(locations)

def compile_dailyMET_Livneh2015_locations(maptable):
    """
    Compile the list of file URLs for Livneh et al., 2015 Daily Meteorology data

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    # gridded data product metadata
    domain = '192.12.137.7'
    subdomain = 'pub/dcp/archive/OBS/livneh2014.1_16deg/ascii/daily'

    locations = []
    for ind, row in maptable.iterrows():
        loci = '_'.join(['Meteorology_Livneh_NAmerExt_15Oct2014', str(row['LAT']), str(row['LONG_'])])
        url = os.path.join('ftp://'+domain, subdomain, 'latitude.'+str(row['LAT']), loci+'.bz2')
        locations.append(url)
    return(locations)

# ### WRF-oriented functions

def compile_wrfnnrp_raw_Salathe2014_locations(maptable):
    """
    Compile a list of file URLs for Salathe et al., 2014 raw WRF NNRP data

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    locations = []
    for ind, row in maptable.iterrows():
        basename = '_'.join(['data', str(row['LAT']), str(row['LONG_'])])
        url = ['http://cses.washington.edu/rocinante/WRF/NNRP/vic_16d/WWA_1950_2010/raw/forcings_ascii/', basename]
        locations.append(''.join(url))
    return(locations)

def compile_wrfnnrp_bc_Salathe2014_locations(maptable):
    """
    Compile a list of file URLs for the Salathe et al., 2014 bias corrected WRF NNRP data

    maptable: (dataframe) a dataframe that contains the FID, LAT, LONG_, and ELEV for each interpolated data file
    """
    locations = []
    for ind, row in maptable.iterrows():
        basename = '_'.join(['data', str(row['LAT']), str(row['LONG_'])])
        url = ['http://cses.washington.edu/rocinante/WRF/NNRP/vic_16d/WWA_1950_2010/bc/forcings_ascii/', basename]
        locations.append(''.join(url))
    return(locations)

# ### Data file migration functions

def ensure_dir(f):
    """
    Check if the folder directory exists, else create it, then set it as the working directory

    f: (dir) the directory to create and/or set as working directory
    """
    if not os.path.exists(f):
        os.makedirs(f)
    os.chdir(f)

def wget_download(listofinterest):
    """
    Download files from an http domain

    listofinterest: (list) a list of urls to request
    """
    # check and download each location point, if it doesn't already exist in the download directory
    for fileurl in listofinterest:
        basename = os.path.basename(fileurl)
        try:
            ping = urllib2.request.urlopen(fileurl)
            if ping.getcode() != 404:
                wget.download(fileurl)
            print('downloaded: ' + basename)
        except:
            print('File does not exist at this URL: ' + basename)

# Download the files to the subdirectory

def wget_download_one(fileurl):
    """
    Download a file from an http domain

    fileurl: (url) a url to request
    """
    # check and download each location point, if it doesn't already exist in the download directory
    basename = os.path.basename(fileurl)

    # if it exists, remove for new download (overwrite mode)
    if os.path.isfile(basename):
        os.remove(basename)

    try:
        ping = urllib2.request.urlopen(fileurl)
        if ping.getcode() != 404:
            wget.download(fileurl)
            print('downloaded: ' + basename)
    except:
        print('File does not exist at this URL: ' + basename)

def wget_download_p(listofinterest, nworkers=20):
    """
    Download files from an http domain in parallel

    listofinterest: (list) a list of urls to request
    nworkers: (int) the number of processors to distribute tasks; default is 20
    """
    # #initialize parallel workers
    # da.set_options(pool=ThreadPool(nworkers))
    # ProgressBar().register()
    # pool = dask.delayed(wget_download_one)(each for each in listofinterest)
    # dask.compute(pool)

    pool = Pool(int(nworkers))
    pool.map(wget_download_one, listofinterest)
    pool.close()
    pool.terminate()

def ftp_download(listofinterest):
    """
    Download and decompress files from an ftp domain

    listofinterest: (list) a list of urls to request
    """
    for loci in listofinterest:
        # establish path info
        fileurl = loci.replace('ftp://', '')  # loci is a url with the domain appended
        ipaddress = fileurl.split('/', 1)[0]  # ip address
        path = os.path.dirname(fileurl.split('/', 1)[1])  # folder path
        filename = os.path.basename(fileurl)  # filename

        # download the file from the ftp server
        ftp = ftplib.FTP(ipaddress)
        ftp.login()
        ftp.cwd(path)
        try:
            ftp.retrbinary('RETR ' + filename, open(filename, 'wb').write)
            ftp.close()

            # decompress the file
            decompbz2(filename)
        except:
            os.remove(filename)
            print('File does not exist at this URL: '+fileurl)

def ftp_download_one(loci):
    """
    Download and decompress a file from an ftp domain

    loci: (url) a url to request
    """
    # establish path info
    fileurl = loci.replace('ftp://', '')  # loci is a url with the domain appended
    ipaddress = fileurl.split('/', 1)[0]  # ip address
    path = os.path.dirname(fileurl.split('/', 1)[1])  # folder path
    filename = os.path.basename(fileurl)  # filename

    # download the file from the ftp server
    ftp = ftplib.FTP(ipaddress)
    ftp.login()
    ftp.cwd(path)
    try:
        ftp.retrbinary('RETR '+filename, open(filename, 'wb').write)
        ftp.close()

        # decompress the file
        decompbz2(filename)
    except:
        os.remove(filename)
        print('File does not exist at this URL: '+fileurl)

def ftp_download_p(listofinterest, nworkers=5):
    """
    Download and decompress files from an ftp domain in parallel

    listofinterest: (list) a list of urls to request
    nworkers: (int) the number of processors to distribute tasks; default is 5
    """
    # #initialize parallel workers
    # da.set_options(pool=ThreadPool(nworkers))
    # ProgressBar().register()
    # pool = dask.delayed(ftp_download_one)(each for each in listofinterest)
    # dask.compute(pool)
    pool = Pool(int(nworkers))
    pool.map(ftp_download_one, listofinterest)
    pool.close()
    pool.terminate()

def decompbz2(filename):
    """
    Extract a file from a bz2 file of the same name, then remove the bz2 file

    filename: (dir) the file path for a bz2 compressed file
    """
    with open(filename.split('.bz2', 1)[0], 'wb') as new_file, open(filename, 'rb') as zipfile:
        decompressor = bz2.BZ2Decompressor()
        for data in iter(lambda: zipfile.read(100 * 1024), b''):
            new_file.write(decompressor.decompress(data))
    os.remove(filename)
    zipfile.close()
    new_file.close()
    # print(os.path.splitext(filename)[0] + ' unzipped')

def catalogfiles(folderpath):
    """
    make a catalog of the gridded files within a folderpath

    folderpath: (dir) the folder of files to be catalogged, which have LAT and LONG_ as the last two filename features
    """
    # read in downloaded files
    temp = [eachfile for eachfile in os.listdir(folderpath) if not os.path.isdir(eachfile)]
    if len(temp) == 0:
        # no files were available; setting default catalog output structure
        catalog = pd.DataFrame([], columns=['filenames', 'LAT', 'LONG_'])
    else:
        # create the catalog dataframe and extract the filename components
        catalog = pd.DataFrame(temp, columns=['filenames'])
        catalog[['LAT', 'LONG_']] = catalog['filenames'].apply(lambda x: pd.Series(str(x).rsplit('_', 2))[1:3]).astype(float)

        # convert the filenames column to a filepath
        catalog['filenames'] = catalog['filenames'].apply(lambda x: os.path.join(folderpath, x))
    return(catalog)

def addCatalogToMap(outfilepath, maptable, folderpath, catalog_label):
    """
    Update the mappingfile with a new column, a vector of filepaths for the downloaded files

    outfilepath: (dir) the path for the output file
    maptable: (dataframe) a dataframe containing the FID, LAT, LONG_, and ELEV information
    folderpath: (dir) the folder of files to be catalogged, which have LAT and LONG_ as the last two filename features
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # assert catalog_label as a string-object
    catalog_label = str(catalog_label)

    # catalog the folder directory
    catalog = catalogfiles(folderpath).rename(columns={'filenames': catalog_label})

    # drop existing column
    if catalog_label in maptable.columns:
        maptable = maptable.drop(labels=catalog_label, axis=1)

    # update with a vector for the catalog of files
    maptable = maptable.merge(catalog, on=['LAT', 'LONG_'], how='left')

    # remove blocks, if they were needed
    if 'blocks' in maptable.columns:
        maptable = maptable.drop(labels=['blocks'], axis=1)

    # write the updated mappingfile
    maptable.to_csv(outfilepath, header=True, index=False)

# Wrapper scripts

def getDailyMET_livneh2013(homedir, mappingfile,
                           subdir='livneh2013/Daily_MET_1915_2011/raw',
                           catalog_label='dailymet_livneh2013', nworkers=4):
    """
    Get the Livneh el al., 2013 Daily Meteorology files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains the LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate DailyMET livneh 2013 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # generate table of lats and long coordinates
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_dailyMET_Livneh2013_locations(maptable)

    # Download the files
    ftp_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

def getDailyMET_livneh2015(homedir, mappingfile,
                           subdir='livneh2015/Daily_MET_1950_2013/raw',
                           catalog_label='dailymet_livneh2015', nworkers=5):
    """
    Get the Livneh el al., 2015 Daily Meteorology files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate Daily MET livneh 2015 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # generate table of lats and long coordinates
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_dailyMET_Livneh2015_locations(maptable)

    # Download the files
    ftp_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

def getDailyMET_bcLivneh2013(homedir, mappingfile,
                             subdir='livneh2013/Daily_MET_1915_2011/bc',
                             catalog_label='dailymet_bclivneh2013', nworkers=20):
    """
    Get the Livneh el al., 2013 bias corrected Daily Meteorology files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate baseline_corrected livneh 2013 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # generate table of lats and long coordinates
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_bc_Livneh2013_locations(maptable)

    # download the files
    wget_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

def getDailyVIC_livneh2013(homedir, mappingfile,
                           subdir='livneh2013/Daily_VIC_1915_2011',
                           catalog_label='dailyvic_livneh2013', nworkers=4):
    """
    Get the Livneh el al., 2013 Daily VIC files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # FIRST RUN
    # check and generate VIC_ASCII Flux model livneh 2013 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # generate table of lats and long coordinates
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_VICASCII_Livneh2013_locations(maptable)

    # Download the files
    ftp_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

def getDailyVIC_livneh2015(homedir, mappingfile,
                           subdir='livneh2015/Daily_VIC_1950_2013',
                           catalog_label='dailyvic_livneh2015', nworkers=5):
    """
    Get the Livneh el al., 2015 Daily VIC files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate Daily VIC.ASCII Flux model livneh 2015 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # generate table of lats and long coordinates
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_VICASCII_Livneh2015_locations(maptable)

    # Download the files
    ftp_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

def getDailyWRF_salathe2014(homedir, mappingfile,
                            subdir='salathe2014/WWA_1950_2010/raw',
                            catalog_label='dailywrf_salathe2014', nworkers=20):
    """
    Get the Salathe el al., 2014 raw Daily WRF files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate the Daily Meteorology raw WRF Salathe 2014 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # read in the longitude and latitude points from the reference mapping file
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_wrfnnrp_raw_Salathe2014_locations(maptable)

    # download the data
    wget_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

def getDailyWRF_bcsalathe2014(homedir, mappingfile,
                              subdir='salathe2014/WWA_1950_2010/bc',
                              catalog_label='dailywrf_bcsalathe2014', nworkers=20):
    """
    Get the Salathe el al., 2014 bias corrected Daily WRF files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate the Daily Meteorology bias corrected WRF Salathe 2014 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # read in the longitude and latitude points from the reference mapping file
    maptable = pd.read_csv(mappingfile)

    # compile the longitude and latitude points
    locations = compile_wrfnnrp_bc_Salathe2014_locations(maptable)

    # download the data
    wget_download_p(locations, nworkers=nworkers)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

    # return to the home directory
    os.chdir(homedir)
    return(filedir)

# # Data Processing libraries

def filesWithPath(folderpath):
    """
    Create a list of filepaths for the files

    folderpath: (dir) the folder of interest
    """
    files = [os.path.join(folderpath, eachfile) for eachfile in os.listdir(folderpath)
            if not eachfile.startswith('.') and not os.path.isdir(eachfile)]  # exclude hidden files
    return(files)

def compareonvar(map_df, colvar='all'):
    """
    subsetting a dataframe based on some columns of interest

    map_df: (dataframe) the dataframe of the mappingfile table
    colvar: (str or list) the column(s) to use for subsetting; 'None' returns an outerjoin, 'all' returns an innerjoin
    """
    # apply row-wise inclusion based on a subset of columns
    if isinstance(colvar, type(None)):
        return(map_df)

    if colvar is 'all':
        # compare on all columns except the station info
        return(map_df.dropna())
    else:
        # compare on only the listed columns
        return(map_df.dropna(subset=colvar))

def mappingfileToDF(mappingfile, colvar='all', summary=True):
    """
    read in a dataframe and subset based on columns of interest

    mappingfile: (dir) the file path to the mappingfile, which contains LAT, LONG_, and ELEV coordinates of interest
    colvar: (str or list) the column(s) to use for subsetting; 'None' returns an outerjoin, 'all' returns an innerjoin
    """
    # Read in the mappingfile as a data frame
    map_df = pd.read_csv(mappingfile)

    # select rows (datafiles) based on the colvar(s) chosen, default is None
    map_df = compareonvar(map_df=map_df, colvar=colvar)

    if summary:
        # compile summaries
        print('Number of gridded data files:' + str(len(map_df)))
        print('Minimum elevation: ' + str(np.min(map_df.ELEV)) + 'm')
        print('Mean elevation: ' + str(np.mean(map_df.ELEV)) + 'm')
        print('Maximum elevation: ' + str(np.max(map_df.ELEV)) + 'm')

    return(map_df, len(map_df))

def read_in_all_files(map_df, dataset, metadata,
                      file_start_date, file_end_date, file_time_step,
                      file_colnames, file_delimiter,
                      subset_start_date, subset_end_date):
    """
    Read in files based on dataset label

    map_df: (dataframe) the mappingfile clipped to the subset that will be read-in
    dataset: (str) the name of the dataset catalogged into map_df
    metadata (str) the dictionary that contains the metadata explanations; default is None
    file_colnames: (list) the list of shorthand variables; default is None
    file_start_date: (date) the start date of the files that will be read-in; default is None
    file_end_date: (date) the end date for the files that will be read in; default is None
    file_time_step: (str) the timedelta code for thedifference between time points; default is 'D' (daily)
    subset_start_date: (date) the start date of a date range of interest
    subset_end_date: (date) the end date of a date range of interest
    """
    # extract metadata if the information are not provided
    if pd.notnull(metadata):

        if isinstance(file_start_date, type(None)):
            file_start_date = metadata[dataset]['start_date']

        if isinstance(file_end_date, type(None)):
            file_end_date = metadata[dataset]['end_date']

        if isinstance(file_time_step, type(None)):
            file_time_step = metadata[dataset]['temporal_resolution']

        if isinstance(file_colnames, type(None)):
            file_colnames = metadata[dataset]['variable_list']

        if isinstance(file_delimiter, type(None)):
            file_delimiter = metadata[dataset]['delimiter']

    # initialize dictionary and time sequence
    df_dict = {}
    met_daily_dates = pd.date_range(file_start_date, file_end_date, freq=file_time_step)  # daily

    # import data for all climate stations
    for ind, row in map_df.iterrows():
        tmp = pd.read_table(row[dataset], header=None, delimiter=file_delimiter, names=file_colnames)
        tmp.set_index(met_daily_dates, inplace=True)

        # subset to the date range of interest (default is file date range)
        tmp = tmp.iloc[(met_daily_dates >= subset_start_date) & (met_daily_dates <= subset_end_date), :]

        # set row indices
        df_dict[tuple(row[['FID', 'LAT', 'LONG_']].tolist())] = tmp

    return(df_dict)

def read_files_to_vardf(map_df, df_dict, gridclimname, dataset, metadata,
                        file_start_date, file_end_date, file_delimiter,
                        file_time_step, file_colnames,
                        subset_start_date, subset_end_date, min_elev, max_elev, variable_list=None):
    """
    # reads in the files to generate variables dataframes

    map_df: (dataframe) the mappingfile clipped to the subset that will be read-in
    df_dict: (dict) an existing dictionary where new computations will be stored
    gridclimname: (str) the suffix for the dataset to be named; if None provided, default to dataset name
    dataset: (str) the name of the dataset catalogged into map_df
    metadata: (str) the dictionary that contains the metadata explanations; default is None
    file_start_date: (date) the start date of the files that will be read-in; default is None
    file_end_date: (date) the end date for the files that will be read in; default is None
    file_delimiter: (str) a file parsing character to be used for file reading
    file_time_step: (str) the timedelta code for the difference between time points; default is 'D' (daily)
    file_colnames: (list) the list of shorthand variables; default is None
    subset_start_date: (date) the start date of a date range of interest
    subset_end_date: (date) the end date of a date range of interest
    min_elev: (float) minimum elevation permitted
    max_elev: (float) maximum elevation permitted
    """
    # start time
    starttime = pd.datetime.now()

    # date range from ogh_meta file
    met_daily_dates = pd.date_range(file_start_date, file_end_date, freq=file_time_step)
    met_daily_subdates = pd.date_range(subset_start_date, subset_end_date, freq=file_time_step)

    # omit null entries or missing data file
    map_df = map_df.loc[pd.notnull(map_df[dataset]), :]
    print('Number of data files within elevation range ({0}-{1} m): {2}'.format(min_elev, max_elev, len(map_df)))

    # establish default list of variables
    if isinstance(variable_list, type(None)):
        variable_list = metadata[dataset]['variable_list']

    # iterate through each data file
    for eachvar in metadata[dataset]['variable_list']:

        # exclude YEAR, MONTH, and DAY
        if eachvar not in ['YEAR', 'MONTH', 'DAY'] and eachvar in variable_list:

            # identify the variable column index
            usecols = [metadata[dataset]['variable_list'].index(eachvar)]

            # initiate df as a list
            df_list = []

            # loop through each file
            for ind, row in map_df.iterrows():

                # consider rewriting the params to just select one column by index at a time
                var_series = da.delayed(pd.read_table)(filepath_or_buffer=row[dataset], delimiter=file_delimiter, header=None, usecols=usecols, names=[tuple(row[['FID', 'LAT', 'LONG_']])])

                # append the series into the list of series
                df_list.append(var_series)

            # concatenate list of series (axis=1 is column-wise) into a dataframe
            df1 = da.delayed(pd.concat)(df_list, axis=1)

            # set and subset date_range index
            df2 = df1.set_index(met_daily_dates, inplace=False).loc[met_daily_subdates]

            # assign dataframe to dictionary object
            df_dict['_'.join([eachvar, gridclimname])] = da.compute(df2)[0]
            print(eachvar+' dataframe reading complete:' + str(pd.datetime.now()-starttime))

    return(df_dict)

def read_daily_streamflow(file_name, drainage_area_m2, file_colnames=None, delimiter='\t', header='infer'):
    """read in a daily streamflow data set"""

    # if file_colnames are supplied, use header=None
    if file_colnames is not None:
        header = None

    # read in the data
    daily_data = pd.read_table(file_name, delimiter=delimiter, header=header)

    # set columns, if header=None
    if file_colnames is not None:
        daily_data.columns = file_colnames
    else:
        file_colnames = list(daily_data.columns)

    # calculate cfs to cms conversion, or vice versa
    if 'flow_cfs' in daily_data.columns:
        flow_cfs = daily_data['flow_cfs']
        flow_cms = flow_cfs/(3.28084**3)
        flow_mmday = flow_cms*1000*3600*24/drainage_area_m2
    elif 'flow_cms' in daily_data.columns:
        flow_cms = daily_data['flow_cms']
        flow_cfs = flow_cms*(3.28084**3)
        flow_mmday = flow_cms*1000*3600*24/drainage_area_m2

    # determine the datetime
    date_index = [file_colnames.index(each) for each in ['year', 'month', 'day']]
    row_dates = pd.to_datetime(daily_data[date_index])

    # generate the daily_flow and set the datetime as row indices
    daily_flow = pd.concat([flow_cfs, flow_cms, flow_mmday], axis=1)
    daily_flow.set_index(row_dates, inplace=True)
    daily_flow.columns = ['flow_cfs', 'flow_cms', 'flow_mmday']
    return(daily_flow)

def read_daily_precip(file_name, file_colnames=None, header='infer', delimiter='\\s+'):
    """read in a daily precipitation data set"""

    # if file_colnames are supplied, use header=None
    if pd.notnull(file_colnames):
        header = None

    # read in the data
    daily_data = pd.read_table(file_name, delimiter=delimiter, header=header)

    # set columns, if header=None
    if pd.notnull(file_colnames):
        daily_data.columns = file_colnames
    else:
        file_colnames = list(daily_data.columns)

    # calculate cfs to cms conversion, or vice versa
    if 'precip_m' in daily_data.columns:
        precip_m = daily_data['precip_m']
        precip_mm = precip_m*1000

    # determine the datetime
    date_index = [file_colnames.index(each) for each in ['year', 'month', 'day']]
    row_dates = pd.to_datetime(daily_data[date_index])

    # generate the daily_flow and set the datetime as row indices
    daily_precip = pd.concat([precip_m, precip_mm], axis=1)
    daily_precip.set_index(row_dates, inplace=True)
    daily_precip.columns = ['precip_m', 'precip_mm']
    return(daily_precip)

def read_daily_snotel(file_name, file_colnames=None, usecols=None, delimiter=',', header='infer'):
    """read in a daily SNOTEL observation data set"""

    # if file_colnames are supplied, use header=None
    if file_colnames is not None:
        header = None

    # read in the data
    daily_data = pd.read_table(file_name, usecols=usecols, header=header, delimiter=delimiter)

    # reset the colnames
    daily_data.columns = ['Date', 'Tmax_C', 'Tmin_C', 'Tavg_C', 'Precip_mm']

    # transform the data
    daily_data['Tmax_C'] = (daily_data['Tmax_C']-32)/1.8
    daily_data['Tmin_C'] = (daily_data['Tmin_C']-32)/1.8
    daily_data['Tavg_C'] = (daily_data['Tavg_C']-32)/1.8
    daily_data['Precip_mm'] = daily_data['Precip_mm']*25.4

    # determine the datetime
    row_dates = pd.to_datetime(daily_data.Date)

    # generate the daily_flow and set the datetime as row indices
    daily_snotel = daily_data[['Tmax_C', 'Tmin_C', 'Tavg_C', 'Precip_mm']]
    daily_snotel.set_index(row_dates, inplace=True)
    return(daily_snotel)

def read_daily_coop(file_name, file_colnames=None, usecols=None, delimiter=',', header='infer'):
    """read in a daily COOP observation data set"""

    # if file_colnames are supplied, use header=None
    if file_colnames is not None:
        header = None

    # read in the data
    daily_data = pd.read_table(file_name, usecols=usecols, header=header, delimiter=delimiter,
                             date_parser=lambda x: pd.datetime.strptime(x, '%Y%m%d'),
                             parse_dates=[0], na_values=-9999)

    # reset the colnames
    daily_data.columns = ['Date', 'Precip_mm', 'Tmax_C', 'Tmin_C', 'Tavg_C']

    # transform the data
    daily_data['Tmax_C'] = (daily_data['Tmax_C']-32)/1.8
    daily_data['Tmin_C'] = (daily_data['Tmin_C']-32)/1.8
    daily_data['Tavg_C'] = (daily_data['Tavg_C']-32)/1.8
    daily_data['Precip_mm'] = daily_data['Precip_mm']*25.4

    # determine the datetime
    row_dates = pd.to_datetime(daily_data.Date)

    # generate the daily_flow and set the datetime as row indices
    daily_coop = daily_data[['Precip_mm', 'Tmax_C', 'Tmin_C', 'Tavg_C']]
    daily_coop.set_index(row_dates, inplace=True)
    return(daily_coop)

# ### Data Processing functions

def generateVarTables(file_dict, gridclimname, dataset, metadata, df_dict=None):
    """
    Slice the files by their common variable

    all_files: (dict) a dictionary of dataframes for each tabular datafile
    dataset: (str) the name of the dataset
    metadata (dict) the dictionary that contains the metadata explanations; default is None
    """
    # combine the files into a pandas panel
    panel = pd.Panel.from_dict(file_dict)

    # initiate output dictionary
    if pd.isnull(df_dict):
        df_dict = dict()

    # slice the panel for each variable in list
    for eachvar in metadata[dataset]['variable_list']:
        df_dict['_'.join([eachvar, gridclimname])] = panel.xs(key=eachvar, axis=2)

    return(df_dict)

# compare two date sets for the start and end of the overlapping dates
def overlappingDates(date_set1, date_set2):
    """
    date_set1: (tuple) a tuple with the start date and end date
    date_set2: (tuple) a tuple with the start date and end date
    """
    # find recent date
    if date_set1[0] > date_set2[0]:
        start_date = date_set1[0]
    else:
        start_date = date_set2[0]

    # find older date
    if date_set1[-1] < date_set2[-1]:
        end_date = date_set1[-1]
    else:
        end_date = date_set2[-1]
    return(start_date, end_date)

def aggregate_space_time_average(df_dict, suffix, start_date, end_date):
    """
    df_dict: (dict) a dictionary to which computed outputs will be stored
    suffix: (str) a string representing the name of the original table
    start_date: (date) the start of the date range within the original table
    end_date: (date) the end of the date range within the original table
    """
    starttime = pd.datetime.now()

    # subset dataframe to the date range of interest
    Var_daily = da.delayed(df_dict[suffix].loc[start_date:end_date, :])

    # Mean daily value at each station
    df_dict['meanbydaily_'+suffix] = pd.DataFrame(Var_daily.mean(axis=0).compute()).T

    # Mean daily value averaged for all stations in analysis
    df_dict['meandaily_'+suffix] = Var_daily.mean(axis=1)

    # Mean monthly value at each station
    df_dict['meanbymonth_'+suffix] = Var_daily.groupby(Var_daily.index.month).mean()

    # Mean monthly value averaged for all stations in analysis
    df_dict['meanmonth_'+suffix] = df_dict['meanbymonth_'+suffix].mean(axis=1)

    # Mean annual value at each station
    df_dict['meanbyyear_'+suffix] = Var_daily.groupby(Var_daily.index.year).mean()

    # mean annual value for each year for all stations in analysis
    df_dict['meanyear_'+suffix] = df_dict['meanbyyear_'+suffix].mean(axis=1)

    # global mean value for all daily values and for all stations in analysis
    df_dict['meanallyear_'+suffix] = df_dict['meandaily_'+suffix].mean(axis=0)

    # annual anomaly compared to the global mean value
    df_dict['anomyear_'+suffix] = df_dict['meanyear_'+suffix] - df_dict['meanallyear_'+suffix]

    df_dict = da.compute(df_dict)[0]
    print(suffix + ' calculations completed in ' + str(pd.datetime.now() - starttime))
    return(df_dict)

def aggregate_space_time_sum(df_dict, suffix, start_date, end_date):
    """
    df_dict: (dict) a dictionary to which computed outputs will be stored
    suffix: (str) a string representing the name of the original table
    start_date: (date) the start of the date range within the original table
    end_date: (date) the end of the date range within the original table
    """
    starttime = pd.datetime.now()

    # subset dataframe to the date range of interest
    Var_daily = da.delayed(df_dict[suffix].loc[start_date:end_date, :])

    # mean daily sum across all stations then averaged across all days in analysis
    df_dict['meanalldailysum_'+suffix] = Var_daily.groupby(pd.TimeGrouper('D')).sum().mean(axis=1)

    # monthly sums for each station and for each month in analysis
    df_dict['monthsum_'+suffix] = Var_daily.groupby(pd.TimeGrouper('M')).sum()

    # mean monthly sum averaged for each stations for each month in analysis
    df_dict['meanbymonthsum_'+suffix] = df_dict['monthsum_'+suffix].groupby(df_dict['monthsum_'+suffix].index.month).mean()

    # mean monthly sum averaged across all stations for each month in analysis
    df_dict['meanmonthsum_'+suffix] = df_dict['meanbymonthsum_'+suffix].mean(axis=1)

    # mean monthly sum averaged across all stations and all months in analysis
    df_dict['meanallmonthsum_'+suffix] = df_dict['meanmonthsum_'+suffix].mean()

    # annual sum for each station and for each year in analysis
    df_dict['yearsum_'+suffix] = Var_daily.groupby(Var_daily.index.year).sum()

    # mean annual sum averaged for each stations across year in analysis
    df_dict['meanbyyearsum_'+suffix] = pd.DataFrame(df_dict['yearsum_'+suffix].mean().compute()).T

    # mean annual sum averaged across all stations for each year in analysis
    df_dict['meanyearsum_'+suffix] = df_dict['yearsum_'+suffix].mean(axis=1)

    # mean annual sum averaged across all stations and all years in analysis
    df_dict['meanallyearsum_'+suffix] = df_dict['meanyearsum_'+suffix].mean()

    df_dict = da.compute(df_dict)[0]
    print(suffix + ' calculations completed in ' + str(pd.datetime.now() - starttime))
    return(df_dict)

def gridclim_dict(mappingfile, dataset, gridclimname=None, metadata=None,
                  variable_list=None, min_elev=None, max_elev=None,
                  file_start_date=None, file_end_date=None, file_time_step=None,
                  file_colnames=None, file_delimiter=None,
                  subset_start_date=None, subset_end_date=None, df_dict=None, colvar=None):
    """
    # pipelined operation for assimilating data, processing it, and standardizing the plotting

    mappingfile: (dir) mapping file path
    dataset: (str) gridded data product shortname as suffix
    gridclimname: (str) user-defined suffix
    metadata: (dict) dictionary of metadata annotations
    variable_list: (list - optional) list of variables to read in
    min_elev: (float64 - optional) min. elevation criteria
    max_elev: (float64 - optional) max. elevation criteria
    file_start_date: (date - optional) time-series start date
    file_end_date: (date - optional) time-series end date
    file_time_step: (str) pandas notation for time-increment
    file_colnames: (list) column names from left to right
    file_delimiter: (str) character to parse columns
    subset_start_date: (date) startdate of analysis
    subset_end_date: (date) enddate of analysis
    df_dict: (dict - optional) existing output dictionary object
    colvar: (str - optional) gridded data product short name for complete file reading
    """
    # generate the climate locations and n_stations
    locations_df, n_stations = mappingfileToDF(mappingfile, colvar=colvar, summary=False)

    # generate the climate station info
    if pd.isnull(min_elev):
        min_elev = locations_df.ELEV.min()

    if pd.isnull(max_elev):
        max_elev = locations_df.ELEV.max()

    # extract metadata if the information are not provided
    if not isinstance(metadata, type(None)):

        if isinstance(file_start_date, type(None)):
            file_start_date = metadata[dataset]['start_date']

        if isinstance(file_end_date, type(None)):
            file_end_date = metadata[dataset]['end_date']

        if isinstance(file_time_step, type(None)):
            file_time_step = metadata[dataset]['temporal_resolution']

        if isinstance(file_colnames, type(None)):
            file_colnames = metadata[dataset]['variable_list']

        if isinstance(file_delimiter, type(None)):
            file_delimiter = metadata[dataset]['delimiter']

    # take all defaults if subset references are null
    if pd.isnull(subset_start_date):
        subset_start_date = file_start_date

    if pd.isnull(subset_end_date):
        subset_end_date = file_end_date

    # initiate output dictionary df_dict was null
    if pd.isnull(df_dict):
        df_dict = {}

    if pd.isnull(gridclimname):
        if pd.notnull(dataset):
            gridclimname = dataset
        else:
            print('no suffix name provided. Provide a gridclimname or dataset label.')
            return

    # assemble the stations within min and max elevantion ranges
    locations_df = locations_df[(locations_df.ELEV >= min_elev) & (locations_df.ELEV <= max_elev)]

    # create dictionary of dataframe
    df_dict = read_files_to_vardf(map_df=locations_df,
                                  dataset=dataset,
                                  metadata=metadata,
                                  variable_list=variable_list,
                                  gridclimname=gridclimname,
                                  file_start_date=file_start_date,
                                  file_end_date=file_end_date,
                                  file_delimiter=file_delimiter,
                                  file_time_step=file_time_step,
                                  file_colnames=file_colnames,
                                  subset_start_date=subset_start_date,
                                  subset_end_date=subset_end_date,
                                  min_elev=min_elev,
                                  max_elev=max_elev,
                                  df_dict=df_dict)

    vardf_list = [eachvardf for eachvardf in df_dict.keys() if eachvardf.endswith(gridclimname)]
    # loop through the dictionary to compute each aggregate_space_time_average object
    for eachvardf in vardf_list:

        # update the dictionary with spatial and temporal average computations
        df_dict.update(aggregate_space_time_average(df_dict=df_dict, suffix=eachvardf,
                                                    start_date=subset_start_date,
                                                    end_date=subset_end_date))

        # if the number of stations exceeds 500, remove daily time-series dataframe
        if len(locations_df) > 300:
            del df_dict[eachvardf]

    return(df_dict)

def compute_diffs(df_dict, df_str, gridclimname1, gridclimname2, prefix1,
                  prefix2='meanmonth', comp_dict=None):
    """
    Compute difference between monthly means (e.g,. Temp) for two different gridded datasets (e.g., Liv, WRF)
    """
    if isinstance(comp_dict, type(None)):
        comp_dict = {}

    for each1 in prefix1:
        for each2 in prefix2:
            diffs = df_dict['_'.join([each2, each1, gridclimname1])] - df_dict['_'.join([each2, each1, gridclimname2])]
            comp_dict['_'.join([str(each1), df_str])] = diffs
    return(comp_dict)

def compute_ratios(df_dict, df_str, gridclimname1, gridclimname2, prefix1,
                   prefix2='meanmonth', comp_dict=None):
    """
    Compute fold-difference between monthly means (e.g,. Temp) for two different gridded datasets (e.g., Liv, WRF)
    """
    if isinstance(comp_dict, type(None)):
        comp_dict = {}

    for each1 in prefix1:
        for each2 in prefix2:
            ratios = df_dict['_'.join([each2, each1, gridclimname1])]/df_dict['_'.join([each2, each1, gridclimname2])]
            comp_dict['_'.join([str(each1), df_str])] = ratios
    return(comp_dict)

def compute_elev_diffs(df_dict, df_str, gridclimname1, prefix1,
                       prefix2a='meanmonth_minelev_', prefix2b='meanmonth_maxelev_'):
    comp_dict = {}
    for each1 in prefix1:
        comp_dict[str(each1) + df_str] = df_dict[prefix2a + each1 + gridclimname1] - df_dict[prefix2b + each1 + gridclimname1]
    return(comp_dict)

def switchUpVICSoil(input_file=None, output_file='soil', mappingfile=None, homedir=None):
    # Read in table of VIC soil inputs -- assumes all Lat/Long set to zero
    soil_base = pd.read_table(input_file, header=None)

    # Make a list of all lat/long values
    latlong = soil_base.apply(lambda x: tuple([x[2], x[3]]), axis=1)

    # Read in mappingfile from TreatGeoSelf()
    maptable = pd.read_table(mappingfile, sep=',')

    # Make a list Lat/Long files that need to switched up
    latlong_1 = maptable.apply(lambda x: tuple([x['LAT'], x['LONG_']]), axis=1)

    # Switch up from 0 to 1 so VIC will run for this Lat/Long point - print new output file (VIC model input file)
    soil_base[0] = latlong.apply(lambda x: 1 if x in set(latlong_1) else 0)
    soil_base.to_csv(output_file, header=False, index=False, sep='\t')
    print(str(soil_base[0].sum()) + ' VIC grid cells have successfully been switched up.')
    print('Check your home directory for your new VIC soil model input set to your list of Lat/Long grid centroids.')

def makebelieve(homedir, mappingfile, BiasCorr, metadata, start_catalog_label, end_catalog_label,
                file_start_date=None, file_end_date=None, data_dir=None, dest_dir_suffix=None):
    np.set_printoptions(precision=6)

    # take liv2013 date set date range as default if file reference dates are not given
    if isinstance(file_start_date, type(None)):
        file_start_date = metadata[start_catalog_label]['start_date']

    if isinstance(file_end_date, type(None)):
        file_end_date = metadata[start_catalog_label]['end_date']

    # generate the month vector
    month = pd.date_range(start=file_start_date, end=file_end_date).month
    month = pd.DataFrame({'month': month})

    # create NEW directory
    if isinstance(dest_dir_suffix, type(None)):
        dest_dir_suffix = 'biascorr_output/'

    dest_dir = os.path.join(homedir, dest_dir_suffix)
    if not os.path.exists(dest_dir):
        os.mkdir(dest_dir)
        print('destdir created')

    # read in the mappingfile
    map_df, nstations = mappingfileToDF(mappingfile, colvar='all', summary=False)

    # compile the BiasCorr dictionary into a pandas panel
    BiasCorr = pd.Panel.from_dict(BiasCorr)

    # loop through each file
    for ind, eachfile in enumerate(map_df.loc[:, start_catalog_label]):

        # identify the file
        station = map_df.loc[map_df.loc[:, start_catalog_label] == eachfile, ['FID', 'LAT', 'LONG_']].reset_index(drop=True)

        # subset the bias correction to the file at hand
        print(str(ind)+' station: '+str(tuple(station.loc[0, :])))
        BiasCorr_df = BiasCorr.xs(key=tuple(station.loc[0, :]), axis=2)

        # read in the file to be corrected
        read_dat = pd.read_table(eachfile, delimiter=metadata[start_catalog_label]['delimiter'],
                                 header=None, names=metadata[start_catalog_label]['variable_list'])

        # extrapolate monthly values for each variable
        for eachvar in read_dat.columns:

            # identify the corresponding bias correction key
            for eachkey in BiasCorr_df.columns:
                if eachkey.startswith(eachvar):

                    # subset the dataframe to the variable in loop
                    BiasCorr_subdf = BiasCorr_df.loc[:, eachkey]

                    # regenerate row index as month column
                    BiasCorr_subdf = BiasCorr_subdf.reset_index().rename(columns={'index': 'month'})

                    # generate the s-vector
                    s = month.merge(BiasCorr_subdf, how='left', on='month').loc[:, eachkey]

                    if eachvar == 'PRECIP':
                        # Use for ratio precip method
                        read_dat[eachvar] = np.multiply(np.array(read_dat.loc[:, eachvar]), np.array(s))
                        # read_dat[eachvar] = np.array(read_dat.loc[:,eachvar])+np.array(s)
                        # positiveprecip=read_dat[eachvar]
                        # positiveprecip[positiveprecip<0.]=0.
                        # read_dat[eachvar] = positiveprecip*.9842
                    else:
                        read_dat[eachvar] = np.array(read_dat.loc[:, eachvar])+np.array(s)

        # write it out to the new destination location
        filedest = os.path.join(dest_dir, os.path.basename(eachfile))
        read_dat.to_csv(filedest, sep='\t', header=None, index=False, float_format='%.4f')

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=map_df, folderpath=dest_dir, catalog_label=end_catalog_label)

    # append the source metadata to the new catalog label metadata
    metadata[end_catalog_label] = metadata[start_catalog_label]

    # update the metadata json file
    json.dump(metadata, open('ogh_meta.json', 'w'), ensure_ascii=False)
    print('mission complete. this device will now self-destruct. just kidding.')
    return(dest_dir, metadata)

def renderWatershed(shapefile, outfilepath, margin=0.25, epsg=4326,
                    basemap_image='Demographics/USA_Social_Vulnerability_Index'):
    """
    shapefile: (dir) the path to the ESRI shapefile for the watershed shape
    outfilepath: (dir) the path for the output image file
    margin: (float) the fraction of width and height to view outside of the watershed shapefile, e.g., 0.25
    epsg: (int) the epsg code for regional projection, e.g. 3857
    basemap_image: (str) the basemap arcgis service e.g., 'Canvas/World_Dark_Gray_Base' or 'ESRI_Imagery_World_2D'
    """
    # generate the figure axis
    fig = plt.figure(figsize=(3, 3), dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # normalize the color distribution according to the value distribution
    cmap = mpl.cm.gnuplot2

    # calculate bounding box based on the watershed shapefile
    watershed = fiona.open(shapefile)
    minx, miny, maxx, maxy = watershed.bounds
    w, h = maxx - minx, maxy - miny
    watershed.close()

    # generate basemap
    m = Basemap(projection='merc', epsg=epsg, resolution='h', ax=ax1,
                llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h, urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)
    m.arcgisimage(service=basemap_image, xpixels=500)

    # read and transform the watershed shapefiles
    m.readshapefile(shapefile=shapefile.replace('.shp', ''), name='watersheds',
                    drawbounds=True, zorder=None, linewidth=0.5, color='m', antialiased=1, default_encoding='utf-8')

    plt.savefig(outfilepath, dpi=500)
    plt.show()
    return(ax1)

def griddedCellGradient(mappingfile, shapefile, outfilepath, plottitle, colorbar_label,
                        spatial_resolution=1/16, margin=0.25, epsg=3857, column='ELEV', polygon_color='m',
                        basemap_image='ESRI_Imagery_World_2D', cmap='coolwarm'):
    """
    mappingfile: (dir) the path to the mappingfile for the watershed gridded cell centroids
    shapefile: (dir) the path to the ESRI shapefile for the watershed shape
    outfilepath: (dir) the path for the output image file
    plottitle: (str) the title of the plot
    colorbar_label: (str) the label for the colorbar
    spatial_resolution: (float) the degree of longitude-latitude separation between gridded cell centroids, e.g., 1/16
    margin: (float) the fraction of width and height to view outside of the watershed shapefile, e.g., 0.25
    epsg: (int) the epsg code for regional projection, e.g. 3857
    column: (str) the name of the column within the mappingfile to visualize with a color gradient
    polygon_color: (str) the colormap code to fill each shapefile polygon; default is 'm' for magenta
    basemap_image: (str) the basemap arcgis service e.g., 'Canvas/World_Dark_Gray_Base' or 'ESRI_Imagery_World_2D'
    cmap: (str) the code for matplotlib colormaps, e.g. 'coolwarm',
    """
    # generate the figure axis
    fig = plt.figure(figsize=(3, 3), dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # read mappingfile, generate gridded cell boxes, and initialize the geodataframe
    map_df = pd.read_csv(mappingfile)
    midpt = spatial_resolution/2
    crs = {'init': 'epsg:{0}'.format(epsg)}
    geometry = map_df.apply(lambda x: box(x['LONG_'] - midpt, x['LAT'] - midpt, x['LONG_'] + midpt, x['LAT'] + midpt), axis=1)
    map_df2 = gpd.GeoDataFrame(map_df, crs=crs, geometry=geometry)

    # normalize the color distribution according to the value distribution
    colormap = mpl.cm.get_cmap(cmap)
    norm = mpl.colors.LogNorm(map_df2[column].min(), map_df2[column].max())
    color_producer = mpl.cm.ScalarMappable(norm=norm, cmap=colormap)

    # calculate bounding box based on the watershed shapefile
    watershed = fiona.open(shapefile)
    minx, miny, maxx, maxy = watershed.bounds
    w, h = maxx - minx, maxy - miny
    watershed.close()

    # generate basemap
    m = Basemap(projection='merc', epsg=epsg, resolution='h', ax=ax1,
                llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h, urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)

    # read and transform the watershed shapefiles
    m.readshapefile(shapefile=shapefile.replace('.shp', ''), name='watershed', drawbounds=True, linewidth=2, color=polygon_color)
    m.arcgisimage(service=basemap_image, xpixels=500)

    # load and transform each polygon in shape
    patches = []
    for ind, eachpol in map_df2.iterrows():
        mpoly = shapely.ops.transform(m, eachpol['geometry'])
        patches.append(PolygonPatch(mpoly, fc=color_producer.to_rgba(eachpol[column]), linewidth=0, alpha=0.5, zorder=5.0))

    # assimilate shapes to plot axis
    coll = PatchCollection(patches, cmap=cmap, match_original=True, zorder=5.0)
    ax1.add_collection(coll)
    coll.set_alpha(0.4)

    # generate colorbar
    coll.set_array(np.array(map_df2[column]))
    cbar = plt.colorbar(coll, shrink=0.5)
    cbar.ax.set_ylabel(colorbar_label, rotation=270, size=3, labelpad=5)  # colorbar label
    cbar.ax.tick_params(labelsize=3)  # colorbar tick fontsize

    # save image
    plt.title(plottitle, fontsize=3)
    plt.savefig(outfilepath, dpi=500)
    plt.show()

def renderValuesInPoints(vardf, vardf_dateindex, shapefile, outfilepath, plottitle, colorbar_label,
                         vmin=None, vmax=None, spatial_resolution=1/16, margin=0.5, gridcell_alpha=0.5, epsg=3857,
                         basemap_image='Canvas/World_Dark_Gray_Base', cmap='coolwarm', figsize=(2, 2)):
    """
    A function to render the dynamics across gridded cell centroids on the spatial landscape

    vardf: (dataframe) a time-series dataframe for a variable with time-points (rows) and gridded cell centroids (column)
    vardf_dateindex: (datetime or float) a datetime identifier to extract a row of data for visualization
    shapefile: (dir) the path to a shapefile
    outfilepath: (dir) the path for the output image file
    plottitle: (str) the title of the plot
    colorbar_label: (str) the label for the colorbar
    spatial_resolution: (float) the degree of longitude-latitude separation between gridded cell centroids, e.g., 1/16
    margin: (float) the fraction of width and height to view outside of the watershed shapefile
    epsg: (int) the epsg code for regional projection, e.g. 3857
    basemap_image: (str) the basemap arcgis service e.g., 'Canvas/World_Dark_Gray_Base' or 'ESRI_Imagery_World_2D'
    cmap: (str) the code for matplotlib colormaps, e.g. 'coolwarm',
    """
    # generate the figure axis
    fig = plt.figure(figsize=figsize, dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # set params
    if isinstance(vmin, type(None)):
        vmin = vardf.values.flatten().min()

    if isinstance(vmax, type(None)):
        vmax = vardf.values.flatten().max()

    # generate the polygon color-scheme
    cmap = mpl.cm.get_cmap(cmap)
    norm = mpl.colors.Normalize(vmin, vmax)
    color_producer = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)

    # calculate bounding box based on the watershed shapefile
    watershed = fiona.open(shapefile)
    minx, miny, maxx, maxy = watershed.bounds
    w, h = maxx - minx, maxy - miny
    watershed.close()

    # generate basemap
    m = Basemap(projection='merc', epsg=epsg, resolution='h', ax=ax1,
                llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h, urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)
    m.arcgisimage(service=basemap_image, xpixels=500)

    # watershed
    m.readshapefile(shapefile=shapefile.replace('.shp', ''), name='watershed', drawbounds=True, linewidth=1, color='m')

    # variable dataframe
    midpt = spatial_resolution/2
    crs = {'init': 'epsg:{0}'.format(epsg)}
    cat = vardf.T.reset_index(level=[1, 2]).rename(columns={'level_1': 'LAT', 'level_2': 'LONG_'})
    geometry = cat.apply(lambda x: shapely.ops.transform(m, box(x['LONG_'] - midpt, x['LAT'] - midpt,
                                                                x['LONG_'] + midpt, x['LAT'] + midpt)), axis=1)
    cat = gpd.GeoDataFrame(cat, crs=crs, geometry=geometry).reset_index(drop=True)

    # geopandas print
    cat.plot(column=vardf_dateindex, cmap=cmap, alpha=gridcell_alpha, ax=ax1, vmin=vmin, vmax=vmax)

    # assimilate the shapes to plot
    patches = []
    for ind, eachpol in cat.iterrows():
        patches.append(PolygonPatch(eachpol['geometry'], linewidth=0, zorder=5.0,
                                    fc=color_producer.to_rgba(eachpol[vardf_dateindex])))

    # assimilate shapes into a patch collection
    coll = PatchCollection(patches, cmap=cmap, match_original=True, zorder=10.0)

    # generate colorbar
    coll.set_array(vardf.values.flatten())
    coll.set_clim([vmin, vmax])
    cbar = plt.colorbar(coll, shrink=0.5)
    cbar.ax.set_ylabel(colorbar_label, rotation=270, size=3, labelpad=3)  # colorbar label
    cbar.ax.tick_params(labelsize=2)  # colorbar tick fontsize
    cbar.outline.set_visible(False)  # colorbar outline

    # save image
    plt.title(plottitle, fontsize=3)
    plt.savefig(outfilepath)
    plt.show()

def findCentroidCode(mappingfile, colvar, colvalue):
    """
    mappingfile: (dir) the file path to the mappingfile, which contains the LAT, LONG_, and ELEV coordinates of interest
    colvar: (string) a column name in mappingfile
    colvalue: (value) a value that corresponds to the colvar column
    """
    mapdf = pd.read_csv(mappingfile)
    outcome = mapdf.loc[mapdf[colvar] == colvalue, :][['FID', 'LAT', 'LONG_']].reset_index(drop=True).set_index('FID')

    return(list(map(tuple, outcome.to_records())))

def mappingfileSummary(listofmappingfiles, listofwatershednames, meta_file):
    """
    Tabulate data availability for all mapping files

    listofmappingfiles: (list) path directories to the mappingfile for each watershed to be compared
    listofwatershednames: (list) strings for the name of each watershed
    """
    datainventory = []

    # loop each mappingfile
    for mappingfile, watershedname in zip(listofmappingfiles, listofwatershednames):
        mapdf = pd.read_csv(mappingfile)

        # summarize the total dimensions
        tmp = []
        tmp.append(tuple(['Watershed', watershedname]))
        tmp.append(tuple(['Median elevation in meters [range](Number of gridded cells)',
                          '{0}[{1}-{2}] (n={3})'.format(int(mapdf.ELEV.median()),
                                                        int(mapdf.ELEV.min()),
                                                        int(mapdf.ELEV.max()),
                                                        int(len(mapdf)))]))

        # summarize for each gridded data product
        for each in mapdf.columns:
            if each in meta_file.keys():
                filesobtained = mapdf[mapdf[each].apply(lambda x: pd.notnull(x))].reset_index()
                if len(filesobtained) > 0:
                    tmp.append(tuple([each,
                                      '{0}[{1}-{2}] (n={3})'.format(int(filesobtained.ELEV.median()),
                                                                    int(filesobtained.ELEV.min()),
                                                                    int(filesobtained.ELEV.max()),
                                                                    int(filesobtained[each].count()))]))

        # interpret list to table form
        t1 = pd.DataFrame.from_records(tmp, columns=['datasets', 'values']).set_index('datasets').T
        t1 = t1.set_index(['Watershed', 'Median elevation in meters [range](Number of gridded cells)'])

        # compile into summary table
        if len(datainventory) == 0:
            datainventory = t1.copy()
        else:
            datainventory = pd.concat([datainventory, t1], axis=0)

    # conform into datasets by watershed summary
    datainventory = datainventory.T.fillna(0)
    datainventory.index.name = None
    return(datainventory)

def dissolveShapefile(listOfShapefiles, listOfNames, newShapefilepath):
    """
    dissolve MultiPolygon Shapefiles into a single shape polygon

    listOfShapefiles: (list) list of shapefile paths
    listOfNames: (list) list of shape names corresponding to the order in listOfShapefiles
    newShapefilepath: (dir) the path to the shapefile of dissolved shapes
    """
    listOfNewShapes = []
    for eachShape, eachName in zip(listOfShapefiles, listOfNames):

        # create dissolved Shapefile destination
        newShapefile = eachShape.replace('.shp', '_2.shp')

        # read shape
        shape = gpd.read_file(eachShape)
        shape['shapeName'] = eachName

        # dissolve shape into new shapefile
        newShape = shape.dissolve(by='shapeName').reset_index()[['shapeName', 'geometry']]
        newShape.to_file(newShapefile)

        listOfNewShapes.append(newShape)

    # concatenate the dissolved shape polygons together
    allShapes = pd.concat(listOfNewShapes, axis=0).reset_index(drop=True)
    allShapes.to_file(newShapefilepath)
    return(allShapes)

def renderValueInBoxplot(vardf, outfilepath, plottitle, time_steps, value_name, cmap,
                         wateryear=False, vmin=None, vmax=None, figsize=(10, 4),
                         reference_lines=False, ref_legend=True, ref_legend_loc=1,
                         obs_datavector=[], obs_datalabel=[], obs_legend=True, obs_legend_loc=2):
    """
    vardf: (dataframe) dataframe of values
    outfilepath: (dir) output file path
    plottitle: (str) title of figure
    time_steps: (month or year) x-axis time-scale
    value_name: (str) y-axis label
    cmap: (str) reference color gradient for colorbar
    wateryear: (logic) organize months using wateryear
    vmin: (float64 - optional) colorbar minimum
    vmax:(float64 - optional) colorbar maximum
    figsize: (tuple) figure height and width in inches
    reference_lines: (list - optional) list of gridded cells to identify as reference lines
    ref_legend: (logic) display reference line legend
    ref_legend_loc: (int) matplotlib code for the legend location
    obs_datavector: (vector) a vector of values to display as dashed lines
    obs_datalabel: (str) the name of the vector
    obs_legend: (logic) display the observation data legend
    obs_legend_loc: (int) matplotlib code for the legend location
    """
    # generate long table
    longtable = pd.melt(vardf.T, value_name=value_name).rename(columns={'variable': time_steps})

    # if the time_steps column are dates, extract month or year
    if isinstance(longtable[time_steps][0], type(pd.datetime.strptime('1900-01-01', '%Y-%m-%d'))):
        if time_steps == 'month':
            longtable[time_steps] = longtable[time_steps].apply(lambda x: x.month)
        elif time_steps == 'year':
            longtable[time_steps] = longtable[time_steps].apply(lambda x: x.year)

    # xaxis order and xaxis labels
    # monthly in wateryear
    if (time_steps == 'month') and (wateryear == True):
        xaxis_order = [10, 11, 12, 1, 2, 3, 4, 5, 6, 7, 8, 9]
        xaxis_labels = [pd.datetime.strptime(str(x), '%m').strftime('%b') for x in xaxis_order]

    # monthly but not wateryear
    elif (time_steps == 'month') and (wateryear == False):
        xaxis_order = sorted(longtable[time_steps].unique())
        xaxis_labels = [pd.datetime.strptime(str(x), '%m').strftime('%b') for x in xaxis_order]

    # annually
    elif (time_steps == 'year'):
        xaxis_order = sorted(longtable[time_steps].unique())
        xaxis_labels = []
        for x in longtable[time_steps].unique():
            if x % 5 == 0:
                xaxis_labels.append(pd.datetime.strptime(str(x), '%Y').strftime('%Y'))
            else:
                xaxis_labels.append(' ')

    # not annually or monthly time_steps - daily or other
    else:
        xaxis_order = sorted(longtable[time_steps].unique())
        xaxis_labels = []
        for ind, x in enumerate(xaxis_order):
            if ind % 10 == 0:
                xaxis_labels.append(str(x))
            else:
                xaxis_labels.append(' ')

    # set params
    if isinstance(vmin, type(None)):
        vmin = longtable[value_name].min()

    if isinstance(vmax, type(None)):
        vmax = longtable[value_name].max()

    # set scalar normalization
    norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)

    # normalize colors
    cm_pdt = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)

    # generate colors
    colors = cm_pdt.to_rgba(longtable.groupby([time_steps])[value_name].median()[xaxis_order])

    # plotting axis
    fig, ax1 = plt.subplots(1, 1, figsize=figsize)

    # apply boxplot with colors
    sns.boxplot(x=time_steps, y=value_name, order=xaxis_order, data=longtable, palette=colors, ax=ax1)
    ax1.set_xlabel(time_steps, fontsize=18)
    ax1.set_ylabel(value_name, fontsize=18)

    # round the yaxis limits
    vrange = vmax - vmin
    plt.ylim(np.round(vmin - (vrange*0.1), 0), np.round(vmax + (vrange*0.1), 0))

    # change ticklabel size
    ax1.xaxis.set_ticklabels(xaxis_labels, fontsize=18, rotation=90)
    ax1.tick_params(labelsize=18)

    # apply observation line plots
    try:
        for ind, (lab, vect) in enumerate(zip(obs_datalabel, obs_datavector)):
            ax1.plot(vect, linestyle='--', linewidth=3, label=lab)
    except:
        pass

    # plot observation legend
    if obs_legend is True:
        vlegend = ax1.legend(loc=obs_legend_loc)
        plt.gca().add_artist(vlegend)

    # apply reference line plots
    try:
        temp_df = vardf.loc[:, reference_lines].loc[xaxis_order, :].reset_index(drop=True)
        t = ax1.plot(temp_df, linewidth=3, zorder=0)
    except:
        pass

    # plot reference legend
    if ref_legend is True:
        tlegend = plt.legend(t, reference_lines, loc=ref_legend_loc)
        plt.gca().add_artist(tlegend)

    # save image
    plt.title(plottitle, fontsize=18)
    plt.savefig(outfilepath)
    plt.show()
    return(ax1)

def multiSiteVisual(listOfShapefiles, listOfNames,
                    multishape='eachwatershed.shp', singleshape='allwatersheds.shp',
                    fileoutpath='annotated_map.png',
                    projection='merc', epsg=3857, polygon_color='m', margin=0.75,
                    scale_x_dist=0, scale_y_dist=-0.25, scale_ref_length=100, scale_yoffset=10000,
                    text_x_dist=0, text_y_dist=0.25, annotate=True):
    """
    Visualize the study site(s)

    listOfShapefiles: (list) a list of paths to shapefile to visualize e.g. [sauk, elwha]
    listOfNames: (list) Site names corresponding with listOfShapefiles e.g., ['Sauk-Suiattle river','Elwha river']
    multishape: (dir) an output shapefile path with each shapefile as a polygon; default is 'eachwatershed.shp'
    singleshape: (dir) an output shapefile path with all polygons dissolved into one; default is 'allwatersheds.shp'
    fileoutpath: (dir) an output file path for the final PNG image; default is 'annotated_map.png'
    projection: (str) the basemap code for the projection; default is 'merc'
    epsg: (int) the EPSG coordinate reference system code; default is 3857
    polygon_color: (str) the colormap code to fill each shapefile polygon; default is 'm' for magenta
    margin: (float) the margin multiplier to set the basemap boundary; default is 0.75 of the height and width
    scale_x_dist: (float) the distance x-degrees from the singleshape centroid to place the mapscale; default is 0
    scale_y_dist: (float) the distance y-degrees from the singleshape centroid to place the mapscale; default is -0.25
    scale_ref_length: (int) the reference distance in km; default is 100
    scale_yoffset: (int) the vertical height of the mapscale in meters; default is 10000 in the projection scale
    text_x_dist: (float) the distance x-degrees from each polygon centroid to place the Site name; default is 0
    text_y_dist: (float) the distance y-degrees from each polygon centroid to place the Site name; default is 0.25
    """
    # inspect each shapefile to be in latlong coordinates
    for eachshp in listOfShapefiles:
        reprojShapefile(sourcepath=eachshp)

    # dissolve shapefiles into a single shapefile containing multiple shape polygons
    w1 = dissolveShapefile(listOfShapefiles=listOfShapefiles, listOfNames=listOfNames, newShapefilepath=multishape)

    # dissolve all shapefiles into a single shapefile containing a single shape polygon
    w2 = w1.copy()
    w2['shapeName'] = 'watershed'
    w2 = w2.dissolve(by='shapeName')
    w2.to_file(singleshape)

    # calculate bounding box based on the watershed shapefile
    minx, miny, maxx, maxy = w2.bounds.iloc[0]
    w, h = maxx - minx, maxy - miny
    center_x, center_y = np.array(w2.centroid.iloc[0])

    # generate the figure axis
    fig = plt.figure(figsize=(3, 3), dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # normalize the color distribution according to the value distribution
    cmap = mpl.cm.gnuplot2

    # generate basemap
    if projection == 'merc':
        m = Basemap(projection='merc', epsg=epsg, resolution='h', ax=ax1,
                    llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h,
                    urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)
    else:
        # center coordinate (for tranverse mercator projections)
        lon0, lat0 = np.array(w2.centroid[0])
        m = Basemap(projection='tmerc', resolution='h', ax=ax1, lat_0=lat0, lon_0=lon0,
                    llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h, urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)

    # affix boundaries
    m.drawcountries(linewidth=0.1)
    m.drawcoastlines(linewidth=0.1)
    m.drawmapboundary(fill_color='lightgray')
    m.fillcontinents(color='white', lake_color='lightgray')
    m.drawrivers(linewidth=0.1, color='lightgray', )
    m.drawstates(linewidth=0.1, color='gray', linestyle='solid')
    m.drawcountries(linewidth=0.1, color='black')

    # read and transform the watershed shapefiles
    m.readshapefile(shapefile=singleshape.replace('.shp', ''), name='allwatersheds', linewidth=0)
    m.readshapefile(shapefile=multishape.replace('.shp', ''), name='eachwatershed', linewidth=0)

    # load and transform each polygon in shape
    patches = [PolygonPatch(Polygon(np.array(shape)), fc=polygon_color, ec=polygon_color, linewidth=0.1, zorder=5.0)
               for info, shape in zip(m.allwatersheds_info, m.allwatersheds)]

    # assimilate shapes to plot axis
    coll = PatchCollection(patches, cmap=cmap, match_original=True, zorder=5.0)
    ax1.add_collection(coll)

    # draw distance scale (coordinate in degrees)
    m.drawmapscale(center_x+scale_x_dist, center_y+scale_y_dist, maxx, maxy,
                   length=scale_ref_length, yoffset=scale_yoffset, barstyle='fancy', fontsize=3, linewidth=0.1)

    # parameters annotated based on non-cyl projections
    if (epsg != 4326) and (annotate is True):

        # annotate watersheds
        for eachinfo, eachpoly in zip(m.eachwatershed_info, m.eachwatershed):
            if (eachinfo['RINGNUM'] == 1):

                # annotate the text in the projection-scaled position
                xycentroid = np.array(Polygon(eachpoly).centroid)
                x0, y0 = m(xycentroid[0], xycentroid[1], inverse=True)
                xytext = np.array(m(x0+text_x_dist, y0+text_y_dist, inverse=False))
                text = eachinfo['shapeName'].replace(' ', '\n')
                plt.annotate(text, fontsize=3, arrowprops=dict(arrowstyle='->'), xy=xycentroid, xytext=xytext)

    # save and show map
    plt.savefig(fileoutpath, dpi=500)
    plt.show()
    return(w1)

def multiSiteStar(listOfShapefiles, listOfNames,
                  multishape='eachwatershed.shp', singleshape='allwatersheds.shp', fileoutpath='annotated_map.png',
                  projection='merc', epsg=3857, polygon_color='m', margin=0.75,
                  scale_x_dist=0, scale_y_dist=-0.25, scale_ref_length=100, scale_yoffset=10000,
                  text_x_dist=0, text_y_dist=0.5):
    """
    Visualize the study site(s)

    listOfShapefiles: (list) a list of paths to shapefile to visualize e.g. [sauk, elwha]
    listOfNames: (list) Site names corresponding with listOfShapefiles e.g., ['Sauk-Suiattle river','Elwha river']
    multishape: (dir) an output shapefile path with each shapefile as a polygon; default is 'eachwatershed.shp'
    singleshape: (dir) an output shapefile path with all polygons dissolved into one; default is 'allwatersheds.shp'
    fileoutpath: (dir) an output file path for the final PNG image; default is 'annotated_map.png'
    projection: (str) the basemap code for the projection; default is 'merc'
    epsg: (int) the EPSG coordinate reference system code; default is 3857
    polygon_color: (str) the colormap code to fill each shapefile polygon; default is 'm' for magenta
    margin: (float) the margin multiplier to set the basemap boundary; default is 0.75 of the height and width
    scale_x_dist: (float) the distance x-degrees from the singleshape centroid to place the mapscale; default is 0
    scale_y_dist: (float) the distance y-degrees from the singleshape centroid to place the mapscale; default is -0.25
    scale_ref_length: (int) the reference distance in km; default is 100
    scale_yoffset: (int) the vertical height of the mapscale in meters; default is 10000 in the projection scale
    text_x_dist: (float) the distance x-degrees from each polygon centroid to place the Site name; default is 0
    text_y_dist: (float) the distance y-degrees from each polygon centroid to place the Site name; default is 0.25
    """
    # inspect each shapefile to be in latlong coordinates
    for eachshp in listOfShapefiles:
        reprojShapefile(sourcepath=eachshp)

    # dissolve shapefiles into a single shapefile containing multiple shape polygons
    w1 = dissolveShapefile(listOfShapefiles=listOfShapefiles, listOfNames=listOfNames, newShapefilepath=multishape)

    # dissolve all shapefiles into a single shapefile containing a single shape polygon
    w2 = w1.copy()
    w2['shapeName'] = 'watershed'
    w2 = w2.dissolve(by='shapeName')
    w2.to_file(singleshape)

    # calculate bounding box based on the watershed shapefile
    minx, miny, maxx, maxy = w2.bounds.iloc[0]
    w, h = maxx - minx, maxy - miny
    center_x, center_y = np.array(w2.centroid.iloc[0])

    # generate the figure axis
    fig = plt.figure(figsize=(3, 3), dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # generate basemap
    if projection == 'merc':
        m = Basemap(projection='merc', epsg=epsg, resolution='h', ax=ax1,
                    llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h, urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)
    else:
        # center coordinate (for tranverse mercator projections)
        lon0, lat0 = np.array(w2.centroid[0])
        m = Basemap(projection='tmerc', resolution='h', ax=ax1, lat_0=lat0, lon_0=lon0,
                    llcrnrlon=minx-margin*w, llcrnrlat=miny-margin*h, urcrnrlon=maxx+margin*w, urcrnrlat=maxy+margin*h)

    # affix boundaries
    m.drawcountries(linewidth=0.1)
    m.drawcoastlines(linewidth=0.1)
    m.drawmapboundary(fill_color='lightgray')
    m.fillcontinents(color='white', lake_color='lightgray')
    m.drawrivers(linewidth=0.1, color='lightgray', )
    m.drawstates(linewidth=0.1, color='gray', linestyle='solid')
    m.drawcountries(linewidth=0.1, color='black')

    # read and transform the watershed shapefiles
    m.readshapefile(shapefile=multishape.replace('.shp', ''), name='eachwatershed', linewidth=0)

    # draw distance scale (coordinate in degrees)
    m.drawmapscale(center_x+scale_x_dist, center_y+scale_y_dist, maxx, maxy,
                   length=scale_ref_length, yoffset=scale_yoffset, barstyle='fancy', fontsize=3, linewidth=0.1)

    # parameters annotated based on non-cyl projections
    if epsg != 4326:
        xs = []
        ys = []
        # annotate watersheds
        for eachinfo, eachpoly in zip(m.eachwatershed_info, m.eachwatershed):
            if (eachinfo['RINGNUM'] == 1):
                # annotate the text in the projection-scaled position
                xycentroid = np.array(Polygon(eachpoly).centroid)
                xs.append(xycentroid[0])
                ys.append(xycentroid[1])

    # assign centroid stars
    m.scatter(xs, ys, s=10, marker=(1, 2, 0), color=polygon_color, cmap=mpl.cm.gnuplot2, alpha=0.7, zorder=10)

    # save and show map
    plt.savefig(fileoutpath, dpi=500)
    plt.show()
    return(w1)

def remapCatalog(homedir, mappingfile, subdir, catalog_label):
    """
    Get the Livneh el al., 2015 Daily Meteorology files of interest using the reference mapping file

    homedir: (dir) the home directory to be used for establishing subdirectories
    mappingfile: (dir) the file path to the mappingfile, which contains the LAT, LONG_, and ELEV coordinates of interest
    subdir: (dir) the subdirectory to be established under homedir
    catalog_label: (str) the preferred name for the series of catalogged filepaths
    """
    # check and generate Daily MET livneh 2015 data directory
    filedir = os.path.join(homedir, subdir)
    ensure_dir(filedir)

    # generate table of lats and long coordinates
    maptable = pd.read_csv(mappingfile)

    # update the mappingfile with the file catalog
    addCatalogToMap(outfilepath=mappingfile, maptable=maptable, folderpath=filedir, catalog_label=catalog_label)

def computeSurfaceArea(shapefile):
    """
    Data-driven computation of surface area using a watershed shapefile

    shapefile: (dir) the path to the study site shapefile for selecting the UTM boundary

    return: (surface area in square meters)
    """

    # ensure projection into WGS84 longlat values
    reprojShapefile(shapefile)

    # generate the figure axis
    fig = plt.figure(figsize=(2, 2), dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # calculate bounding box based on the watershed shapefile
    watershed = gpd.read_file(shapefile)
    watershed['watershed'] = 'watershed'
    watershed = watershed.dissolve(by='watershed')

    # extract area centroid, bounding box info, and dimension shape
    lon0, lat0 = np.array(watershed.centroid.iloc[0])
    minx, miny, maxx, maxy = watershed.bounds.iloc[0]

    # generate traverse mercatur projection
    m = Basemap(projection='tmerc', resolution='l', ax=ax1, lat_0=lat0, lon_0=lon0,
                llcrnrlon=minx, llcrnrlat=miny, urcrnrlon=maxx, urcrnrlat=maxy)

    # apply UTM transformation
    geometry = watershed['geometry'].apply(lambda x: shapely.ops.transform(m, x))

    # compute gridded cell area
    surfacearea = np.array(geometry.apply(lambda x: x.area))
    plt.gcf().clear()
    return(surfacearea)

def computeGCSurfaceArea(shapefile, spatial_resolution, vardf):
    """
    Data-driven computation of gridded cell surface area using the list of gridded cells centroids

    shapefile: (dir) the path to the study site shapefile for selecting the UTM boundary
    spatial_resolution: (float) the spatial resolution in degree coordinate reference system e.g., 1/16
    vardf: (dataframe) input dataframe that contains FID, LAT and LONG references for each gridded cell centroid

    return: (mean surface area in meters-squared, standard deviation in surface area)
    """

    # ensure projection into WGS84 longlat values
    reprojShapefile(shapefile)

    # generate the figure axis
    fig = plt.figure(figsize=(2, 2), dpi=500)
    ax1 = plt.subplot2grid((1, 1), (0, 0))

    # calculate bounding box based on the watershed shapefile
    watershed = gpd.read_file(shapefile)
    watershed['watershed'] = 'watershed'
    watershed = watershed.dissolve(by='watershed')

    # extract area centroid, bounding box info, and dimension shape
    lon0, lat0 = np.array(watershed.centroid.iloc[0])
    minx, miny, maxx, maxy = watershed.bounds.iloc[0]

    # generate traverse mercatur projection
    m = Basemap(projection='tmerc', resolution='l', ax=ax1, lat_0=lat0, lon_0=lon0,
                llcrnrlon=minx, llcrnrlat=miny, urcrnrlon=maxx, urcrnrlat=maxy)

    # generate gridded cell bounding boxes
    midpt_dist = spatial_resolution/2
    cat = vardf.T.reset_index(level=[1, 2]).rename(columns={'level_1': 'LAT', 'level_2': 'LONG_'})
    geometry = cat.apply(lambda x:
                         shapely.ops.transform(m, box(x['LONG_']-midpt_dist, x['LAT']-midpt_dist,
                                                      x['LONG_']+midpt_dist, x['LAT']+midpt_dist)), axis=1)

    # compute gridded cell area
    gc_area = geometry.apply(lambda x: x.area)
    plt.gcf().clear()
    return(gc_area.mean(), gc_area.std())

def cfs_to_mmday(cfs, SA_sq_ft):
    """
    cfs: (float) flow rate in cubic feet per second
    SA_sq_ft: (float) surface area in square feet
    """
    return(cfs/SA_sq_ft * 24 * 60 * 60 * 304.8)  # cfs / sqft * s/min * min*hr * hr/day * mm/ft

def sec_to_day(sec):
    day = sec*24*60*60
    return(day)

def cms_to_cfs(cms):
    cfs = cms*(3.28084**3)
    return(cfs)

def in_to_mm(inch):
    mm = inch*25.4
    return(mm)

def F_to_C(F):
    C = (32*F - 32) * (5/9)
    return(C)

def monthlyExceedence_cfs(df_dict, daily_streamflow_dfname, gridcell_area, exceedance):
    """
    df_dict: (dict) dictionary of spatial-temporal computation dataframes
    daily_streamflow_dfname: (str) name of daily streamflow dataframe in df_dict in millimeters per second (mm/s)
    gridcell_area: (float) the estimated surface area of the gridded area in square meters
    exceedance: (float) the percent exceedance probability
    """

    # dataset name
    dataset = daily_streamflow_dfname.split('_', 1)[1]

    # convert mm_s to m_s
    mm_s = df_dict[daily_streamflow_dfname]
    m_s = mm_s * 0.001

    # multiply streamflow (mps) with grid cell surface area (m2) to produce volumetric streamflow (cms)
    cms = m_s.multiply(np.array(gridcell_area))

    # convert m^3/s to cfs; multiply with (3.28084)^3
    cfs = cms.multiply((3.28084)**3)

    # output to df_dict
    df_dict['cfs_' + daily_streamflow_dfname] = cfs

    months = range(1, 13)
    Exceed = pd.DataFrame()

    # for each month
    for ind, eachmonth in enumerate(months):
        month_res = cfs.iloc[cfs.index.month == eachmonth, :].apply(lambda x: np.percentile(x, 100 - exceedance), axis=0)
        Exceed = pd.concat([Exceed, pd.DataFrame(month_res).T], axis=0)

    Exceed.index = months
    df_dict['EXCEED{0}_{1}'.format(exceedance, dataset)] = Exceed
    return(df_dict)

def monthlyExceedence_mmday(df_dict, daily_streamflow_dfname, exceedance):
    """
    df_dict: (dict) dictionary of spatial-temporal computation dataframes
    daily_streamflow_dfname: (str) name of daily streamflow dataframe in df_dict in millimeters per second (mm/s)
    gridcell_area: (float) the estimated surface area of the gridded area in square meters
    exceedance: (float) the percent exceedance probability
    """
    # dataset name
    dataset = daily_streamflow_dfname.split('_', 1)[1]

    # initialize the streamflow in mm_day
    mmday = df_dict[daily_streamflow_dfname]

    months = range(1, 13)
    Exceed = pd.DataFrame()

    # for each month
    for ind, eachmonth in enumerate(months):
        month_res = mmday.iloc[mmday.index.month == eachmonth, :].apply(lambda x: np.percentile(x, 100 - exceedance), axis=0)
        Exceed = pd.concat([Exceed, pd.DataFrame(month_res).T], axis=0)

    Exceed.index = months
    df_dict['EXCEED{0}_mmday_{1}'.format(exceedance, dataset)] = Exceed
    return(df_dict)


import whois
from pprint import pprint

domains = '''
aipa.ca
split5.ca

'''

#domains = ''

for d in domains.split('\n'):
	if d:
		print('-'*80)
		print(d)
		w = whois.query(d, ignore_returncode=1)
		if w:
			wd = w.__dict__
			for k, v in wd.items():
				print('%20s\t"%s"' % (k, v))


import os, pathlib

# SOURCE_PATH = 'D:/Pictures/All/Special/'
SOURCE_PATH = 'D:/Pictures/'

def printContents(directory):
    dirCount = 0
    fileCount = 0
    for root, dirs, files in os.walk(directory):
        level = root.replace(directory, '').count(os.sep)
        indent = "    " * level
        print(f"{indent}{pathlib.PureWindowsPath(root)}")
        dirCount += 1
        fileCount += len(files)
        for file in files:
            print(f"{indent}    {file}")
    print('{} directories (including root) and {} files'.format(dirCount, fileCount))

printContents(SOURCE_PATH)