#!/usr/bin/env python
#
# Participants Editor - DEI-FCTUC IHC 2011
#
# Developers:
#	- Renato Rodrigues
#	- Rui Molar
#
from models import *
from xml.dom.minidom import Document
from google.appengine.ext import webapp

def createParticipantsXml(self):
		#Get Participants from DB
		providers = Provider.all().order('provider_name')
		consumers = Consumer.all().order('profile_name')
		partners = Partner.all().order('partner_name')

		#Start create XML file
		doc = Document()
		#Create Root
		root = doc.createElement("Participants")
		doc.appendChild(root)

		#List of Providers
		listProvider = doc.createElement("Providers")
		root.appendChild(listProvider)

		#Populate XML file
		for tempProvider in providers:
			#Create XML Elements
			provider = doc.createElement("provider")
			provider.setAttribute("key",str(tempProvider.key()))
			listProvider.appendChild(provider)
			providerKey = doc.createElement("providerKey")
			providerName = doc.createElement("providerName")
			organizationField = doc.createElement("organization")
			name = doc.createElement("name")
			www = doc.createElement("www")
			address = doc.createElement("address")
			street = doc.createElement("street")
			postcode = doc.createElement("postcode")
			city = doc.createElement("city")
			state = doc.createElement("state")
			country = doc.createElement("country")
			contactPerson = doc.createElement("contactPerson")
			contactName = doc.createElement("contactName")
			contactPhone = doc.createElement("contactPhone")
			contactEmail = doc.createElement("contactEmail")
			description = doc.createElement("description")
			#Structure of XML File
			provider.appendChild(providerKey)
			provider.appendChild(providerName)
			provider.appendChild(organizationField)
			organizationField.appendChild(name)
			organizationField.appendChild(www)
			organizationField.appendChild(address)
			address.appendChild(street)
			address.appendChild(postcode)
			address.appendChild(city)
			address.appendChild(state)
			address.appendChild(country)
			provider.appendChild(contactPerson)
			contactPerson.appendChild(contactName)
			contactPerson.appendChild(contactPhone)
			contactPerson.appendChild(contactEmail)
			provider.appendChild(description)

			#Get Provider Key
			key=doc.createTextNode(tempProvider.provider_key)
			providerKey.appendChild(key)
			#Get Provider Name
			pname=doc.createTextNode(tempProvider.provider_name)
			providerName.appendChild(pname)
			#Get Organization
			org=doc.createTextNode(tempProvider.organization)
			name.appendChild(org)

			#Get More Info About Organization
			tempOrg = Organization.all().filter('name = ',tempProvider.organization).get()
			txt = doc.createTextNode(tempOrg.website)
			www.appendChild(txt)
			#Address
			tempStreet = doc.createTextNode(tempOrg.street)
			street.appendChild(tempStreet)
			tempPostcode = doc.createTextNode(tempOrg.post_code)
			postcode.appendChild(tempPostcode)
			tempCity = doc.createTextNode(tempOrg.city)
			city.appendChild(tempCity)
			tempState = doc.createTextNode(tempOrg.state)
			state.appendChild(tempState)
			tempCountry = doc.createTextNode(tempOrg.country)
			country.appendChild(tempCountry)

			#Get More Info About Contact
			div=tempProvider.contact_person.split(" ")
			firstName=div[0]
			lastName=div[1]
			tempContact = Contact.all()
			tempContact.filter('first_name = ',firstName)
			tempContact.filter('last_name = ',lastName)
			tempContact=tempContact.get()

			tempName = doc.createTextNode(tempProvider.contact_person)
			contactName.appendChild(tempName)
			tempPhone = doc.createTextNode(tempContact.telephone)
			contactPhone.appendChild(tempPhone)
			tempEmail = doc.createTextNode(tempContact.email)
			contactEmail.appendChild(tempEmail)

			desc = doc.createTextNode(tempProvider.description)
			description.appendChild(desc)

		#List Consumers
		listConsumer = doc.createElement("Consumers")
		root.appendChild(listConsumer)

		#Populate XML file
		for tempConsumer in consumers:
			#Create XML Elements
			consumer = doc.createElement("consumer")
			consumer.setAttribute("key",str(tempConsumer.key()))
			listConsumer.appendChild(consumer)

			profileName = doc.createElement("profileName")
			problemDefinition = doc.createElement("problemDefinition")
			goalDefinition = doc.createElement("goalDefinition")
			description = doc.createElement("description")

			#Structure of XML File
			consumer.appendChild(profileName)
			consumer.appendChild(problemDefinition)
			consumer.appendChild(goalDefinition)
			consumer.appendChild(description)

			#Get Consumer Name
			name=doc.createTextNode(tempConsumer.profile_name)
			profileName.appendChild(name)
			#Get Problem Definition
			pd=doc.createTextNode(tempConsumer.problem_definition)
			problemDefinition.appendChild(pd)
			#Get Goal Definition
			gd=doc.createTextNode(tempConsumer.goal_definition)
			goalDefinition.appendChild(gd)
			#Get Description
			desc = doc.createTextNode(tempConsumer.description)
			description.appendChild(desc)

		#List Partners
		listPartners = doc.createElement("Partners")
		root.appendChild(listPartners)

		#Populate XML file
		for tempPartner in partners:
			#Create XML Elements
			partner = doc.createElement("partner")
			partner.setAttribute("key",str(tempPartner.key()))
			listPartners.appendChild(partner)

			partnerKey = doc.createElement("partnerKey")
			partnerName = doc.createElement("partnerName")
			partnerType = doc.createElement("partnerType")
			organizationField = doc.createElement("organization")
			name = doc.createElement("name")
			www = doc.createElement("www")
			address = doc.createElement("address")
			street = doc.createElement("street")
			postcode = doc.createElement("postcode")
			city = doc.createElement("city")
			state = doc.createElement("state")
			country = doc.createElement("country")
			contactPerson = doc.createElement("contactPerson")
			contactName = doc.createElement("contactName")
			contactPhone = doc.createElement("contactPhone")
			contactEmail = doc.createElement("contactEmail")
			description = doc.createElement("description")
			#Structure of XML File
			partner.appendChild(partnerKey)
			partner.appendChild(partnerName)
			partner.appendChild(partnerType)
			partner.appendChild(organizationField)
			organizationField.appendChild(name)
			organizationField.appendChild(www)
			organizationField.appendChild(address)
			address.appendChild(street)
			address.appendChild(postcode)
			address.appendChild(city)
			address.appendChild(state)
			address.appendChild(country)
			partner.appendChild(contactPerson)
			contactPerson.appendChild(contactName)
			contactPerson.appendChild(contactPhone)
			contactPerson.appendChild(contactEmail)
			partner.appendChild(description)

			#Get Partner Key
			key=doc.createTextNode(tempPartner.partner_key)
			partnerKey.appendChild(key)
			#Get Partner Name
			pname=doc.createTextNode(tempPartner.partner_name)
			partnerName.appendChild(pname)
			#Get Partner Type
			ptype=doc.createTextNode(tempPartner.partner_type)
			partnerType.appendChild(ptype)
			#Get Organization
			org=doc.createTextNode(tempPartner.organization)
			name.appendChild(org)

			#Get More Info About Organization
			tempOrg = Organization.all().filter('name = ',tempPartner.organization).get()
			txt = doc.createTextNode(tempOrg.website)
			www.appendChild(txt)
			#Address
			tempStreet = doc.createTextNode(tempOrg.street)
			street.appendChild(tempStreet)
			tempPostcode = doc.createTextNode(tempOrg.post_code)
			postcode.appendChild(tempPostcode)
			tempCity = doc.createTextNode(tempOrg.city)
			city.appendChild(tempCity)
			tempState = doc.createTextNode(tempOrg.state)
			state.appendChild(tempState)
			tempCountry = doc.createTextNode(tempOrg.country)
			country.appendChild(tempCountry)

			#Get More Info About Contact
			div=tempPartner.contact_person.split(" ")
			firstName=div[0]
			lastName=div[1]
			tempContact = Contact.all()
			tempContact.filter('first_name = ',firstName)
			tempContact.filter('last_name = ',lastName)
			tempContact=tempContact.get()

			tempName = doc.createTextNode(tempPartner.contact_person)
			contactName.appendChild(tempName)
			tempPhone = doc.createTextNode(tempContact.telephone)
			contactPhone.appendChild(tempPhone)
			tempEmail = doc.createTextNode(tempContact.email)
			contactEmail.appendChild(tempEmail)

			desc = doc.createTextNode(tempPartner.description)
			description.appendChild(desc)

		#Print/Send XML file
		#print doc.toprettyxml(indent="  ")
		self.response.headers["Content-Type"] = "application/xml"
		self.response.out.write(doc.toprettyxml(indent="  "))


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 11 12:09:02 2022

Class and function used for handling data files with mixed UBX data from ublox GNSS modules and frm LDS70A Laser altimeter.

@author: Laktop
"""

import numpy as np
import re
import math
from datetime import datetime
import matplotlib.pyplot as pl
import os,sys
from cmcrameri import cm
from geopy import distance
import cartopy.crs as ccrs
import cartopy.io.img_tiles as cimgt
from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter
import io
from urllib.request import urlopen, Request
from PIL import Image

# %%  data class

class MSG_type:

    def __init__(self,MSG):
        self.len=0
        self.MSG=MSG

    def addData(self,keys,values):
        self.len=(len(values))
        if len(keys)!=len(values[0]):
            print("Keys and values don't have same length!!")
            print(values[0])
            print(keys)

            return
        self.keys=keys.copy()
        for i,k in enumerate(keys):
            setattr(self,k,np.array([l[i] for l in values]))

_MSG_list_=['Laser','PINS1','PSTRB','PINS2']    # NMEA message list to parse
_keyList_=[['h','signQ','T','TOW'],
          ['TOW','GPSWeek','insStatus','hdwStatus','roll','pitch','heading','velX', 'velY', 'velZ','lat', 'lon', 'height','OffsetLLA_N','OffsetLLA_E','OffsetLLA_D'],
          ['GPSWeek','TOW','pin','count'],
          ['TOW','GPSWeek','insStatus','hdwStatus','QuatW','QuatX','QuatY','QuatZ','velX', 'velY', 'velZ','lat', 'lon', 'height']]  # order matters!!

class INSLASERdata:

    # extr_list=['PINS1','Laser']

    def __init__(self,filepath,name='',load=True, droplaserTow0=True,
                 correct_Laser=True,distCenter=0, pitch0=0, roll0=0,laser_time_offset=0,c_pitch=1,c_roll=1):
        """
            Read GNSS and Laser data from .ubx data file. Additional methods are available for plotting and handling data.

            Inputs:
            ---------------------------------------------------
            filepath:           file path

        """

        if name!='':
            self.name=name
        else:
             self.name=filepath.split('\\')[-1]

        self.filepath=filepath
        self.ToW=0
        self.distCenter=distCenter
        self.pitch0=pitch0
        self.roll0=roll0
        self.c_pitch=c_pitch
        self.c_roll=c_roll

        self.keyList=_keyList_.copy()
        self.MSG_list=_MSG_list_.copy()

        if load:
            self.loadData(correct_Laser=correct_Laser,droplaserTow0=droplaserTow0)

    def loadData(self,correct_Laser=0,droplaserTow0=True):
        if not os.path.isfile(self.filepath) :
            print("File not found!!!")
            raise FileNotFoundError()

        print('Reading file: ',self.filepath)
        print('----------------------------')
        file = open(self.filepath, 'rt')

        for msg in (self.MSG_list):
            setattr(self,msg+'List',[])

        i=0

        self.corrupt=[]
        self.other=[]
        self.dropped=[]

        for l in file:
            # print(l)
            # l2=str(l)
            i+=1
            if l[0]=='#':
                continue
            elif l.find('$')!=-1:
                Msg_key,data=parseNMEAfloat(l)

                if Msg_key=='PINS1':
                    self.ToW=data[0]

                if Msg_key=='Error':
                    self.corrupt.append(l)
                    # print("Message {:s} not in NMEA message list".format(Msg_key))
                    try:
                        self.dropped.index(Msg_key)
                    except ValueError:
                        self.dropped.append(Msg_key)
                    continue

                try:
                  j=self.MSG_list.index(Msg_key)
                  if len(data)!=len(self.keyList[j]):
                      self.corrupt.append(l)
                      continue
                except ValueError:
                    print("Message {:s} not in NMEA message list. Dropping it.".format(Msg_key))
                    continue

                getattr(self,Msg_key+'List').append(data)

            elif l.find('D ')!=-1:
                self.LaserList.append(parseLaser(l)+(self.ToW,) )

            else:
                self.other.append(l)

        # drop  laser points with ToW=0
        if droplaserTow0:
            for j,l in enumerate(self.LaserList):
                if l[-1]!=0:
                    break
            self.LaserList=self.LaserList[j:]

        for msg in (self.MSG_list):
            setattr(self,msg,MSG_type(msg) )
            if len(getattr(self,msg+'List'))>0:
                values=np.array(getattr(self,msg+'List'))
                keys=self.keyList[self.MSG_list.index(msg)]
                print(keys)
                getattr(self,msg).addData(keys,values)

            # delattr(self,msg+'List')

        print("Total lines read: ", i)

        # correct h with angles from INS
        if correct_Laser:
                self.corr_h_laser()

    def corr_h_laser(self):
        """
        correct height with angles from INS
        """
        try:
            i=self.PINS1.TOW.searchsorted( self.Laser.TOW)
            j=np.array(i)-1

            pitch=self.PINS1.pitch[j]+(self.PINS1.pitch[i]-self.PINS1.pitch[j])/(self.PINS1.TOW[i]-self.PINS1.TOW[j])*(self.Laser.TOW-self.PINS1.TOW[j])
            roll=self.PINS1.roll[j]+(self.PINS1.roll[i]-self.PINS1.roll[j])/(self.PINS1.TOW[i]-self.PINS1.TOW[j])*(self.Laser.TOW-self.PINS1.TOW[j])

            roll-=self.roll0
            pitch-=self.pitch0
            # roll*=self.c_roll
            # pitch*=self.c_pitch

            self.Laser.pitch=pitch
            self.Laser.roll=roll
            self.Laser.h_corr=self.Laser.h*(np.cos(pitch)*np.cos(roll))-self.distCenter*np.sin(pitch)
            self.Laser.keys.extend(['h_corr','roll','pitch'])

        except Exception as e:
            print(e)
            print('Failed to correct Laser height')
            try:
                self.Laser.h_corr=np.zeros_like(self.Laser.h)
            except AttributeError:
                print('Laser data not found.')

    def plot_att(self,ax=[]):
        plot_att(self,ax=ax)

    def plot_elevation_time(self,ax=[],title=[]):
        plot_elevation_time(self,ax=ax,title=title)

    def plot_longlat(self,z='height',ax=[],cmap= cm.batlow):
        plot_longlat(self,z=z,ax=ax,cmap=cmap)

    def plot_map(self,z='height',ax=[],cmap= cm.batlow):
        plot_map(self,z=z,ax=ax,cmap=cmap)

    def plot_mapOSM(self,z='TOW',ax=[],cmap= cm.batlow,title=[],extent=[]):
        plot_mapOSM(self,z=z,ax=ax,cmap= cmap,title=title,extent=extent)

    def subset(self, timelim=[],timeformat='s'):
        """
        Parameters
        ----------
        timelim : List, optional
            limits in time. The default is [].
        timeformat : string, optional
            units of time limits. Either 'TOW' or 's' (relative to start of data).

        Returns
        -------
        data2 : UBX2data object with subset of data into time limits

        """
        d=self.PINS1
        if timeformat=='TOW':
              lim=d.TOW.searchsorted(timelim)
              TOW_lim=timelim
        elif timeformat=='s':
              lim=((d.TOW -d.TOW[0]) ).searchsorted(timelim)
              TOW_lim=d.TOW[lim]
        else:
            print('timeformat not valid')
            return 0

        data2=INSLASERdata(self.filepath,name=self.name,load=False)

        for attr in (self.MSG_list):
            # print(attr)
            try:
                msg_data=getattr(self,attr)
                lim2=msg_data.TOW.searchsorted(TOW_lim)
            except Exception as e:
                    print(e)
                    print(attr)
                    # continue
            d2=MSG_type(attr)

            for a in msg_data.__dict__.keys():
                # print('\t',a)
                try:
                    setattr(d2,a,np.array(getattr(msg_data, a)[lim2[0]:lim2[1]] ))
                except:
                    setattr(d2,a,getattr(msg_data, a))
            setattr(d2,'len',lim2[1]-lim2[0])
            setattr(data2,attr,d2)
        return data2

# %% #########function definitions #############

def parseNMEA(l):
    """
    l: string with data
    TOW: time of Week to append to message data

    return height, signal quality, temperature

    """

    Msg_key=l[l.find('$')+1:l.find(',') ]
    start=l.find('$'+Msg_key)
    end=l.find('*')

    # Check if message is valid
    if (start!=-1 and end!=-1 and start<end and chksum_nmea(l[start:end+3])):

      try:
          a=np.array(l[start+len(Msg_key)+2:end].split(','),dtype=float)

      except:
          try:
              a=np.array(l[start+len(Msg_key)+2:end].split(','))
          except:
              print('Coud not parse valid NMEA string:', l)
              return 'Error',l
    else:
        print('Coud not parse string:', l)
        return 'Error',l

    return Msg_key,a

def parseNMEAfloat(l):
    """
    l: string with data
    TOW: time of Week to append to message data

    return height, signal quality, temperature

    """

    Msg_key=l[l.find('$')+1:l.find(',') ]
    start=l.find('$'+Msg_key)
    end=l.find('*')

    # Check if message is valid
    if (start!=-1 and end!=-1 and start<end and chksum_nmea(l[start:end+3])):

      try:
          a=np.array(l[start+len(Msg_key)+2:end].split(','),dtype=float)

      except:
              print('Coud not parse valid NMEA string:', l)
              return 'Error',l
    else:
        print('Coud not parse string:', l)
        return 'Error',l

    return Msg_key,a

def parseLaser(l):
    """
    l: string with data
    TOW: time of Week to append to message data

    return height, signal quality, temperature

    """

    h=[]
    T=[]
    signQ=[]
    TOW=[]

    i=0
    j=0
    t=0
    length=len(l)

    if l[0]=='D':

        a=l.split()
        if len(a)==2:
            signQ= np.nan
            T= np.nan
            try:
                h= float(a[1])
            except:
                h= np.nan
                print('coud not parse string:', l)
        else:
            error=0
            try:
                h= float(a[1])
            except:
                h= np.nan
                error=1
            try:
                signQ= float(a[2])
            except:
                signQ= np.nan
                error=1
            try:
                T= float(a[3])
            except:
                T= np.nan
                error=1
            if error:
                print('coud not parse string:', l)
    else:
        h= np.nan
        signQ= np.nan
        T= np.nan
        print('coud not parse string:', l)

    return h,signQ,T

def chksum_nmea(sentence):
    # From: http://doschman.blogspot.com/2013/01/calculating-nmea-sentence-checksums.html

    # This is a string, will need to convert it to hex for
    # proper comparsion below
    end=sentence.find("*")
    cksum = sentence[end+1:end+3]

    # String slicing: Grabs all the characters
    # between '$' and '*' and nukes any lingering
    # newline or CRLF
    chksumdata = re.sub("(\n|\r\n)","", sentence[sentence.find("$")+1:sentence.find("*")])

    # Initializing our first XOR value
    csum = 0

    # For each char in chksumdata, XOR against the previous
    # XOR'd char.  The final XOR of the last char will be our
    # checksum to verify against the checksum we sliced off
    # the NMEA sentence

    for c in chksumdata:
       # XOR'ing value of csum against the next char in line
       # and storing the new XOR value in csum
       csum ^= ord(c)

    # Do we have a validated sentence?
    try:
        if hex(csum) == hex(int(cksum, 16)):
           return True
    except  ValueError:
        print('Invalid checksum: ',cksum)
    return False

def check_data(data):
    """
    Check data and produce some plots

    Input:
    --------------
    data        Member of class UBXdata

    """

    print('\n\n###############################\n-----------------------------\n',
          data.name,
          '\n----------------------------\n###############################\n')

    for attr in data.MSG_list:
        try:
            d=getattr(data,attr)
            print('\n',attr,'\n----------------------------')
            print('Length:')
            print(d.len)
            print('Time intervall (s):')
            print((d.TOW[:5]-d.TOW[0]) )
        except Exception as e:
                print(e)

    print('\nOthers: \n----------------------------')
    print('Length:')
    try:
        print(len(data.other))
    except:
        print('no data')
    # for c in data.other[:10]:
    #     print(c)

    print('\nCorrupted: \n----------------------------')
    try:
        print(len(data.corrupt))
    except:
        print('no data')
    # for c in data.corrupt[:10]:
    #     print(c)

    try:
        data.plot_att()
    except AttributeError:
        print('no attitude messages found')

    try:
        if len(data.Laser.h)==0:
            raise AttributeError

        pl.figure()
        ax=pl.subplot(111)
        ax2=ax.twinx()
        ax.set_ylabel('Laser h (m)')
        ax2.set_ylabel('GNSS Delta_h (m)')
        ax.set_xlabel('time (ms)')

        ax.plot(data.Laser.TOW-data.PINS1.TOW[0],data.Laser.h,'--xk',label='Laser')
        # ax.plot(data.Laser.TOW2-data.PINS1.TOW[0],data.Laser.h,'--ob',label='Laser, time2')
        ax2.plot((data.PINS1.TOW-data.PINS1.TOW[0]),data.PINS1.height -(data.PINS1.height[0] -data.Laser.h[0]),'+:r',label='GPS height')

        lines, labels = ax.get_legend_handles_labels()
        lines2, labels2 = ax2.get_legend_handles_labels()
        ax2.legend(lines + lines2, labels + labels2, loc=0)
        pl.title(data.name)
    except AttributeError:
        print('No laser data found')

def plot_att(data,ax=[],title='',heading=True):
    if ax==[]:
        fig=pl.figure()
        ax=pl.subplot(111)
    else:
        pl.sca(ax)

    d=data.PINS1

    if heading:
            ax2=ax.twinx()

    ax.plot((d.TOW-d.TOW[0]),np.degrees(d.pitch),'o-r',label='pitch')
    ax.plot((d.TOW-d.TOW[0]),np.degrees(d.roll),'x-k',label='roll')
    if heading:
        ax2.plot((d.TOW-d.TOW[0]),np.degrees(d.heading),'x-b',label='heading')

    ax.set_ylabel('pitch/roll (deg)')
    ax.set_xlabel('time (s)')
    if heading:
        ax2.set_ylabel('heading (deg)')
        # ask matplotlib for the plotted objects and their labels
        lines, labels = ax.get_legend_handles_labels()
        lines2, labels2 = ax2.get_legend_handles_labels()
        ax2.legend(lines + lines2, labels + labels2, loc=0)
    else:
        ax.legend(loc=0)

    if title=='':
        pl.title(data.name)
    elif title!='none':
        pl.title(title)

def plot_att_laser(data,ax=[],title='',heading=True):
    if ax==[]:
        fig=pl.figure()
        ax=pl.subplot(111)
    else:
        pl.sca(ax)
        fig=pl.gcf()

    d=data.PINS1

    if heading:
            ax2=ax.twinx()

    ax.plot((d.TOW-d.TOW[0]),np.degrees(d.pitch),'o-r',label='pitch')
    ax.plot((d.TOW-d.TOW[0]),np.degrees(d.roll),'x-k',label='roll')
    if heading:
        ax2.plot((d.TOW-d.TOW[0]),np.degrees(d.heading),'x-b',label='heading')

    ax.set_ylabel('pitch/roll (deg)')
    ax.set_xlabel('time (s)')
    if heading:
        ax2.set_ylabel('heading (deg)')
        # ask matplotlib for the plotted objects and their labels
        lines, labels = ax.get_legend_handles_labels()
        lines2, labels2 = ax2.get_legend_handles_labels()
        ax2.legend(lines + lines2, labels + labels2, loc=0)
        ax2.yaxis.label.set_color('b')
        ax2.tick_params(axis='y', colors='b')
    else:
        ax.legend(loc=0)

    try:
        if len(data.Laser.h)==0:
            raise AttributeError

        ax3=ax.twinx()
        fig.subplots_adjust(right=0.75)
        ax3.spines['right'].set_position(("axes", 1.2))
        ax3.yaxis.label.set_color('g')
        ax3.tick_params(axis='y', colors='g')
        ax3.set_ylabel('Laser h (m)')
        ax3.plot((data.Laser.TOW-d.TOW[0]),data.Laser.h,':og',label='Laser')
        lines3, labels3 = ax3.get_legend_handles_labels()
    except AttributeError:
        print('No laser data found')
        lines3=[]
        labels3=[]

    if title=='':
        pl.title(data.name)
    elif title!='none':
        pl.title(title)

def plot_elevation_time(data,ax=[],title=[]):
    if ax==[]:
        fig=pl.figure()
        ax=pl.subplot(111)
    else:
        pl.sca(ax)

    d=data.PINS1

    ax.plot((d.TOW-d.TOW[0]),d.height,'o-r',label='height')
    ax.set_ylabel('elevation (m a.s.l.)')
    ax.set_xlabel('time (s)')

    if title:
        pl.title(title)
    pl.tight_layout()

def plot_longlat(data,z='height',ax=[],cmap= cm.batlow):
    if ax==[]:
        fig=pl.figure()
        ax=pl.subplot(111)
    else:
        pl.sca(ax)

    d=data.PINS1

    ax2=ax.twinx()
    c=getattr(d,z)
    if z=='height':
        label='elevation (m a.s.l.)'
        c=c
    elif z=='TOW':
        label='time (s)'
        c-=c[0]
        c=c
    else:
        label=z
    lat=np.array([distance.distance((d.lat[0],d.lon[0]),(x,d.lon[0])).m for x in d.lat ])
    lon=np.array([distance.distance((d.lat[0],d.lon[0]),(d.lat[0],x)).m for x in d.lon ])

    im=ax.scatter(lon,lat,c=c,cmap=cmap,marker='x')
    c=pl.colorbar(im, label=label)
    ax.invert_xaxis()
    ax.set_ylabel('N (m)')
    ax.set_xlabel('W (m)')

    pl.title(data.name)
    pl.tight_layout()

def plot_summary(data,extent,cmap=cm.batlow,heading=True):

    fig=pl.figure(figsize=(8,10))
    spec = fig.add_gridspec(ncols=1, nrows=9)
    cimgt.OSM.get_image = image_spoof # reformat web request for street map spoofing
    osm_img = cimgt.OSM() # spoofed, downloaded street map

    ax0 = fig.add_subplot(spec[0:3, 0],projection=osm_img.crs)
    data.plot_mapOSM(z='TOW',extent=extent,ax=ax0, cmap=cmap  )

    ax1 = fig.add_subplot(spec[3:5, 0])
    ax1.plot((data.PINS1.TOW-data.PINS1.TOW[0]) ,
              [distance.distance((data.PINS1.lat[0],data.PINS1.lon[0]),(data.PINS1.lat[0],x)).m for x in data.PINS1.lon ],
              'x:',label='East-West')
    ax1.plot((data.PINS1.TOW-data.PINS1.TOW[0]) ,
              [distance.distance((data.PINS1.lat[0],data.PINS1.lon[0]),(x,data.PINS1.lon[0])).m for x in data.PINS1.lat ],
              '+:',label='North-South')
    ax1.set_ylabel('Distance (m)')
    ax1.legend()

    ax2 = fig.add_subplot(spec[5:7, 0],sharex = ax1)
    ax2.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.h, 'x:',label='original')
    ax2.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.h_corr, '+:k',label='corrected')
    ax2.plot((data.PINS1.TOW-data.PINS1.TOW[0]) ,data.PINS1.height -(data.PINS1.height[0] -data.Laser.h[0]),'+:r',label='GPS height')
    ax2.set_ylabel('h_laser (m)')
    ax2.set_xlim(ax1.get_xlim())
    ax2.legend()

    ax3 = fig.add_subplot(spec[7:9, 0],sharex = ax1)
    plot_att(data,ax=ax3,title='none',heading=heading)
    pl.tight_layout()
    return fig

def laser_correction(data,show_corr_angles=0,GPS_h=False,heading=False):

    fig, [ax2,ax3] = pl.subplots(2, 1, figsize=(8, 8), sharex=True, sharey=False)

    ax2.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.h, 'x:',label='original')
    ax2.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.h_corr, '+:',label='corrected')
    if GPS_h:
        ax2.plot((data.PINS1.TOW-data.PINS1.TOW[0]) ,data.PINS1.height -(data.PINS1.height[0] -data.Laser.h[0]),'+:r',label='GPS height')
    ax2.set_ylabel('h_laser (m)')
    ax2.legend()

    plot_att(data,ax=ax3,title='none',heading=heading)

    if show_corr_angles:
        ax3.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,np.degrees(data.Laser.pitch), 'x:',label='pitch laser')
        ax3.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,np.degrees(data.Laser.roll), 'x:',label='roll laser')
        ax3.legend(loc=0)
    return fig

def laser_correction_superimposed(data,GPS_h=False):

    fig, ax2 = pl.subplots(1, 1, figsize=(8, 8))

    ax2.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.h, 'x:',label='original')
    ax2.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.h_corr, '+:',label='corrected')
    if GPS_h:
        ax2.plot((data.PINS1.TOW-data.PINS1.TOW[0]) ,data.PINS1.height -(data.PINS1.height[0] -data.Laser.h[0]),'+:r',label='GPS height')
    ax2.set_ylabel('h_laser (m)')
    ax2.legend(loc=2)

    ax3=ax2.twinx()
    ax3.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.pitch, 'x:k',label='pitch laser')
    ax3.plot((data.Laser.TOW-data.PINS1.TOW[0]) ,data.Laser.roll, '+:r',label='roll laser')
    ax3.legend(loc=1)
    ax3.set_ylabel('Angle (rad)')
    ax2.grid()
    return fig

# %% plot on map
def plot_map(data,z='height',ax=[],cmap= cm.batlow,title=[],timelim=[],timeformat='ms'):

    if ax==[]:
        fig=pl.figure()
        ax = pl.axes(projection=ccrs.PlateCarree())
    else:
        ax.axes(projection=ccrs.PlateCarree())

    d=data.PINS1
    if not timelim==[]:
        if timeformat=='ms':
            lim=d.TOW.searchsorted(timelim)
        if timeformat=='s':
            lim=((d.TOW -d.TOW[0]) ).searchsorted(timelim)

    c=getattr(d,z)[lim[0]:lim[1]] # get data color plot

    if z=='height':
        label='elevation (m a.s.l.)'
        c=c
    elif z=='TOW':
        label='time (s)'
        c-=c[0]
        c=c
    else:
        label=z

    ax.coastlines()
    im=ax.scatter(d.lon[lim[0]:lim[1]],d.lat[lim[0]:lim[1]],c=c,cmap=cmap,marker='x')
    c=pl.colorbar(im, label=label)

    lon_formatter = LongitudeFormatter(number_format='0.1f',degree_symbol='',dateline_direction_label=True) # format lons
    lat_formatter = LatitudeFormatter(number_format='0.1f',degree_symbol='') # format lats
    ax.xaxis.set_major_formatter(lon_formatter) # set lons
    ax.yaxis.set_major_formatter(lat_formatter) # set lats

    if title:
        pl.title(title)
    pl.tight_layout()

def image_spoof(self, tile): # this function pretends not to be a Python script
        url = self._image_url(tile) # get the url of the street map API
        req = Request(url) # start request
        req.add_header('User-agent','Anaconda 3') # add user agent to request
        fh = urlopen(req)
        im_data = io.BytesIO(fh.read()) # get image
        fh.close() # close url
        img = Image.open(im_data) # open image with PIL
        img = img.convert(self.desired_tile_form) # set image format
        return img, self.tileextent(tile), 'lower' # reformat for cartopy

def plot_mapOSM(data,z='height',ax=[],cmap= cm.batlow,title=[], extent=[]):
    """
    Plot data (z) on Open Street Map layer.

    Parameters
    ----------
    data : TYPE
        DESCRIPTION.

    z : TYPE, optional
        DESCRIPTION. The default is 'height'.
    ax : TYPE, optional
        DESCRIPTION. The default is [].
    cmap : TYPE, optional
        DESCRIPTION. The default is cm.batlow.
    title : TYPE, optional
        DESCRIPTION. The default is [].
    extent:     limits of map. In [[lon,lon,lat,lat]]

    Returns
    -------
    None.

    """
    cimgt.OSM.get_image = image_spoof # reformat web request for street map spoofing
    osm_img = cimgt.OSM() # spoofed, downloaded street map

    if ax==[]:
        fig=pl.figure()
        ax = pl.axes(projection=osm_img.crs)

    # prepare data
    d=data.PINS1
    c=np.array( getattr(d,z))
    if z=='height':
        label='elevation (m a.s.l.)'
        c=c
    elif z=='TOW':
        label='time (s)'
        c-=c[0]
        c=c
    else:
        label=z

    if extent==[]:
        dx=(np.max(d.lon)-np.min(d.lon))
        dy=(np.max(d.lat)-np.min(d.lat))
        W=np.max([dx,dy])*3/5
        center=[(np.max(d.lon)+np.min(d.lon))/2, (np.max(d.lat)+np.min(d.lat))/2]
        extent = [center[0]-W,center[0]+W, center[1]-W,center[1]+W] # Contiguous US bounds
        print(extent)

    # setup map
    ax.set_extent(extent) # set extents
    ax.set_xticks(np.linspace(extent[0],extent[1],3),crs=ccrs.PlateCarree()) # set longitude indicators
    ax.set_yticks(np.linspace(extent[2],extent[3],4)[1:],crs=ccrs.PlateCarree()) # set latitude indicators
    lon_formatter = LongitudeFormatter(number_format='0.4f',degree_symbol='',dateline_direction_label=True) # format lons
    lat_formatter = LatitudeFormatter(number_format='0.4f',degree_symbol='') # format lats
    ax.xaxis.set_major_formatter(lon_formatter) # set lons
    ax.yaxis.set_major_formatter(lat_formatter) # set lats
    ax.xaxis.set_tick_params()
    ax.yaxis.set_tick_params()

    scale = np.ceil(-np.sqrt(2)*np.log(np.divide((extent[1]-extent[0])/2.0,350.0))) # empirical solve for scale based on zoom
    scale = (scale<20) and scale or 19 # scale cannot be larger than 19
    ax.add_image(osm_img, int(scale))

    im=ax.scatter(d.lon,d.lat,c=c,cmap=cmap,marker='x',transform=ccrs.PlateCarree())
    c=pl.colorbar(im, label=label)

    if title:
        pl.title(title)
    pl.tight_layout()

def getextent(data):
    minlon=data.PINS1.lon.min()
    maxlon=data.PINS1.lon.max()
    minlat=data.PINS1.lat.min()
    maxlat=data.PINS1.lat.max()
    dlat=maxlat-minlat
    dlon=maxlon-minlon
    return minlon-dlon/10,maxlon+dlon/10,minlat-dlat/10,maxlat+dlat/10


import torch
import torch.nn as nn
from loss.ssim import SSIM

class SegmentationLosses(object):
    def __init__(self, weight=None,  batch_average=True, cuda=False):
        self.weight = weight
        self.batch_average = batch_average
        self.cuda = cuda

    def build_loss(self, mode='ce'):
        """Choices: ['ce' or 'focal']"""
        if mode == 'ce':
            return self.CrossEntropyLoss
        elif mode == 'focal':
            return self.FocalLoss
        elif mode == 'ssim':
            return self.SSIMLoss
        else:
            raise NotImplementedError

    def CrossEntropyLoss(self, logit, target):
        n, c, h, w = logit.size()
        criterion = nn.CrossEntropyLoss(weight=self.weight,
                                        reduction='mean')
        if self.cuda:
            criterion = criterion.cuda()

        loss = criterion(logit, target.long())

        if self.batch_average:
            loss /= n

        return loss

    def FocalLoss(self, logit, target, gamma=2, alpha=0.5): #0.5
        n, c, h, w = logit.size()
        criterion = nn.CrossEntropyLoss(weight=self.weight,
                                        reduction='mean')
        if self.cuda:
            criterion = criterion.cuda()

        logpt = -criterion(logit, target.long())
        pt = torch.exp(logpt)
        if alpha is not None:
            logpt *= alpha
        loss = -((1 - pt) ** gamma) * logpt

        if self.batch_average:
            loss /= n

        return loss

    def SSIMLoss(self, logit, target):
        n, c, h, w = logit.size()
        criterion = SSIM(window_size=11)
        if self.cuda:
            criterion = criterion.cuda()

        loss = criterion(logit, target.long())

        if self.batch_average:
            loss /= n

        return 1-loss


import os, sys, inspect
import numpy as np
from copy import deepcopy
from ..mfbase import MFDataException

def clean_name(name):
    # remove bad characters
    clean_string = name.replace(' ', '_')
    clean_string = clean_string.replace('-', '_')
    # remove anything after a parenthesis
    index = clean_string.find('(')
    if index != -1:
        clean_string = clean_string[0:index]

    return clean_string

def find_keyword(arr_line, keyword_dict):
    # convert to lower case
    arr_line_lower = []
    for word in arr_line:
        # integers and floats are not keywords
        if not DatumUtil.is_int(word) and not DatumUtil.is_float(word):
            arr_line_lower.append(word.lower())
    # look for constants in order of most words to least words
    key = ''
    for num_words in range(len(arr_line_lower), -1, -1):
        key = tuple(arr_line_lower[0:num_words])
        if len(key) > 0 and key in keyword_dict:
            return key
    return None

def max_tuple_abs_size(some_tuple):
    max_size = 0
    for item in some_tuple:
        item_abs = abs(item)
        if item_abs > max_size:
            max_size = item_abs
    return max_size

class TemplateGenerator(object):
    """
    Abstract base class for building a data template for different data types.
    This is a generic class that is initialized with a path that identifies
    the data to be built.

    Parameters
    ----------
    path : string
        tuple containing path of data is described in dfn files
        (<model>,<package>,<block>,<data name>)
    """
    def __init__(self, path):
        self.path = path

    def _get_data_dimensions(self, model):
        from ..data import mfstructure
        from ..coordinates import modeldimensions

        # get structure info
        sim_struct = mfstructure.MFStructure().sim_struct
        package_struct = sim_struct.get_data_structure(self.path[0:-2])

        # get dimension info
        data_struct = sim_struct.get_data_structure(self.path)
        package_dim = modeldimensions.PackageDimensions([model.dimensions],
                                                        package_struct,
                                                        self.path[0:-1])
        return data_struct, modeldimensions.DataDimensions(package_dim,
                                                           data_struct)

    def build_type_header(self, type, data=None):
        from ..data.mfdata import DataStorageType

        if type == DataStorageType.internal_array:
            if isinstance(self, ArrayTemplateGenerator):
                return {'factor':1.0, 'iprn':1, 'data':data}
            else:
                return None
        elif type == DataStorageType.internal_constant:
            return data
        elif type == DataStorageType.external_file:
            return {'filename':'', 'factor':1.0, 'iprn':1}
        return None

class ArrayTemplateGenerator(TemplateGenerator):
    """
    Class that builds a data template for MFArrays.  This is a generic class
    that is initialized with a path that identifies the data to be built.

    Parameters
    ----------
    path : string
        tuple containing path of data is described in dfn files
        (<model>,<package>,<block>,<data name>)

    Methods
    -------
    empty: (model: MFModel, layered: boolean, data_storage_type_list: boolean,
            default_value: int/float) : variable
        Builds a template for the data you need to specify for a specific data
        type (ie. "hk") in a specific model.  The data type and dimensions
        is determined by "path" during initialization of this class and the
        model is passed in to this method as the "model" parameter.  If the
        data is transient a dictionary containing a single stress period
        will be returned.  If "layered" is set to true, data will be returned
        as a list ndarrays, one for each layer.  data_storage_type_list is a
        list of DataStorageType, one type for each layer.  If "default_value"
        is specified the data template will be populated with that value,
        otherwise each ndarray in the data template will be populated with
        np.empty (0 or 0.0 if the DataStorageType is a constant).
    """
    def __init__(self, path):
        super(ArrayTemplateGenerator, self).__init__(path)

    def empty(self, model=None, layered=False, data_storage_type_list=None,
              default_value=None):
        from ..data import mfdata, mfstructure
        from ..data.mfdata import DataStorageType

        # get the expected dimensions of the data
        data_struct, data_dimensions = self._get_data_dimensions(model)
        datum_type = data_struct.get_datum_type()
        data_type = data_struct.get_datatype()
        # build a temporary data storge object
        data_storage = mfdata.DataStorage(
                model.simulation_data, data_dimensions, None,
                mfdata.DataStorageType.internal_array,
                mfdata.DataStructureType.recarray)
        dimension_list = data_storage.get_data_dimensions(None)

        # if layered data
        if layered and dimension_list[0] > 1:
            if data_storage_type_list is not None and \
                    len(data_storage_type_list) != dimension_list[0]:
                comment = 'data_storage_type_list specified with the ' \
                          'wrong size.  Size {} but expected to be ' \
                          'the same as the number of layers, ' \
                          '{}.'.format(len(data_storage_type_list),
                                       dimension_list[0])
                type_, value_, traceback_ = sys.exc_info()

                raise MFDataException(data_struct.get_model(),
                                      data_struct.get_package(),
                                      data_struct.path,
                                      'generating array template',
                                      data_struct.name,
                                      inspect.stack()[0][3],
                                      type_, value_, traceback_, comment,
                                      model.simulation_data.debug)
            # build each layer
            data_with_header = []
            for layer in range(0, dimension_list[0]):
                # determine storage type
                if data_storage_type_list is None:
                    data_storage_type = DataStorageType.internal_array
                else:
                    data_storage_type = data_storage_type_list[layer]
                # build data type header
                data_with_header.append(self._build_layer(datum_type,
                                                          data_storage_type,
                                                          default_value,
                                                          dimension_list))
        else:
            if data_storage_type_list is None or \
                    data_storage_type_list[0] == \
                            DataStorageType.internal_array:
                data_storage_type = DataStorageType.internal_array
            else:
                data_storage_type = data_storage_type_list[0]
            # build data type header
            data_with_header = self._build_layer(datum_type,
                                                 data_storage_type,
                                                 default_value,
                                                 dimension_list, True)

        # if transient/multiple list
        if data_type == mfstructure.DataType.array_transient:
            # Return as dictionary
            return {0:data_with_header}
        else:
            return data_with_header

    def _build_layer(self, data_type, data_storage_type, default_value,
                     dimension_list, all_layers=False):
        from ..data.mfdata import DataStorageType

        # build data
        if data_storage_type == DataStorageType.internal_array:
            if default_value is None:
                if all_layers:
                    data = np.empty(dimension_list, data_type)
                else:
                    data = np.empty(dimension_list[1:], data_type)
            else:
                if all_layers:
                    data = np.full(dimension_list, default_value, data_type)
                else:
                    data = np.full(dimension_list[1:], default_value,
                                   data_type)
        elif data_storage_type == DataStorageType.internal_constant:
            if default_value is None:
                if data_type == np.int:
                    data = 0
                else:
                    data = 0.0
            else:
                data = default_value
        else:
            data = None
        # build data type header
        return self.build_type_header(data_storage_type, data)

class ListTemplateGenerator(TemplateGenerator):
    """
    Class that builds a data template for MFLists.  This is a generic class
    that is initialized with a path that identifies the data to be built.

    Parameters
    ----------
    path : string
        tuple containing path of data is described in dfn files
        (<model>,<package>,<block>,<data name>)

    Methods
    -------
    empty: (maxbound: int, aux_vars: list, boundnames: boolean, nseg: int) :
            dictionary
        Builds a template for the data you need to specify for a specific data
        type (ie. "stress_period_data") in a specific model.  The data type is
        determined by "path" during initialization of this class.  If the data
        is transient a dictionary containing a single stress period will be
        returned.  The number of entries in the recarray are determined by
        the "maxbound" parameter.  The "aux_vars" parameter is a list of aux
        var names to be used in this data list.  If boundnames is set to
        true and boundname field will be included in the recarray.  nseg is
        only used on list data that contains segments.  If timeseries is true,
        a template that is compatible with time series data is returned.
    """
    def __init__(self, path):
        super(ListTemplateGenerator, self).__init__(path)

    def _build_template_data(self, type_list):
        template_data = []
        for type in type_list:
            if type[1] == int:
                template_data.append(0)
            elif type[1] == float:
                template_data.append(np.nan)
            else:
                template_data.append(None)
        return tuple(template_data)

    def empty(self, model, maxbound=None, aux_vars=None, boundnames=False,
              nseg=None, timeseries=False, stress_periods=None):
        from ..data import mfdata, mfstructure

        data_struct, data_dimensions = self._get_data_dimensions(model)
        data_type = data_struct.get_datatype()
        # build a temporary data storge object
        data_storage = mfdata.DataStorage(
                model.simulation_data, data_dimensions, None,
                mfdata.DataStorageType.internal_array,
                mfdata.DataStructureType.recarray)

        # build type list
        type_list = data_storage.build_type_list(nseg=nseg)
        if aux_vars is not None:
            if len(aux_vars) > 0 and (isinstance(aux_vars[0], list) or
                    isinstance(aux_vars[0], tuple)):
                aux_vars = aux_vars[0]
            for aux_var in aux_vars:
                type_list.append((aux_var, object))
        if boundnames:
            type_list.append(('boundnames', object))

        if timeseries:
            # fix type list to make all types objects
            for index in range(0, len(type_list)):
                type_list[index] = (type_list[index][0], object)

        # build rec array
        template_data = self._build_template_data(type_list)
        rec_array_data = []
        if maxbound is not None:
            for index in range(0, maxbound):
                rec_array_data.append(template_data)
        else:
            rec_array_data.append(template_data)
        rec_array = np.rec.array(rec_array_data, type_list)

        # if transient/multiple list
        if data_type == mfstructure.DataType.list_transient or \
                data_type == mfstructure.DataType.list_multiple:
            # Return as dictionary
            if stress_periods is None:
                return {0:rec_array}
            else:
                template = {}
                for stress_period in stress_periods:
                    template[stress_period] = deepcopy(rec_array)
                return template
        else:
            return rec_array

class DatumUtil(object):
    @ staticmethod
    def is_int(str):
        try:
            int(str)
            return True
        except TypeError:
            return False
        except ValueError:
            return False

    @ staticmethod
    def is_float(str):
        try:
            float(str)
            return True
        except TypeError:
            return False
        except ValueError:
            return False

class ArrayUtil(object):
    """
    Class contains miscellaneous methods to work with and compare arrays

    Parameters
    ----------
    path : string
        file path to read/write to
    max_error : float
        maximum acceptable error when doing a compare of floating point numbers

    Methods
    -------
    is_empty_list : (current_list : list) : boolean
        determines if an n-dimensional list is empty
    con_convert : (data : string, data_type : type that has conversion
                   operation) : boolean
        returns true if data can be converted into data_type
    max_multi_dim_list_size : (current_list : list) : boolean
        determines the max number of items in a multi-dimensional list
        'current_list'
    first_item : (current_list : list) : variable
        returns the first item in the list 'current_list'
    next_item : (current_list : list) : variable
        returns the next item in the list 'current_list'
    array_comp : (first_array : list, second_array : list) : boolean
        compares two lists, returns true if they are identical (with max_error)
    spilt_data_line : (line : string) : list
        splits a string apart (using split) and then cleans up the results
        dealing with various MODFLOW input file releated delimiters.  returns
        the delimiter type used.
    clean_numeric : (text : string) : string
        returns a cleaned up version of 'text' with only numeric characters
    save_array_diff : (first_array : list, second_array : list,
                       first_array_name : string, second_array_name : string)
        saves lists 'first_array' and 'second_array' to files first_array_name
        and second_array_name and then saves the difference of the two
        arrays to 'debug_array_diff.txt'
    save_array(filename : string, multi_array : list)
        saves 'multi_array' to the file 'filename'
    """
    numeric_chars = {'0': 0, '1': 0, '2': 0, '3': 0, '4': 0, '5': 0,
                     '6': 0, '7': 0, '8': 0, '9': 0, '.': 0, '-': 0}
    quote_list = {"'", '"'}
    delimiter_list = {',': 0, '\t': 0, ' ': 0}
    delimiter_used = None
    line_num = 0
    consistent_delim = False

    def __init__(self, path=None, max_error=0.01):
        self.max_error = max_error
        if path:
            self.path = path
        else:
            self.path = os.getcwd()

    @staticmethod
    def has_one_item(current_list):
        if not isinstance(current_list, list) and not isinstance(current_list,
                                                                 np.ndarray):
            return True
        if len(current_list) != 1:
            return False
        if (isinstance(current_list[0], list) or
                isinstance(current_list, np.ndarray)) and \
                len(current_list[0] != 0):
            return False
        return True

    @staticmethod
    def is_empty_list(current_list):
        if not isinstance(current_list, list):
            return not current_list

        for item in current_list:
            if isinstance(item, list):
                # still in a list of lists, recurse
                if not ArrayUtil.is_empty_list(item):
                    return False
            else:
                return False

        return True

    @staticmethod
    def max_multi_dim_list_size(current_list):
        max_length = -1
        for item in current_list:
            if len(item) > max_length:
                max_length = len(item)
        return max_length

    @staticmethod
    def first_item(current_list):
        if not isinstance(current_list, list):
            return current_list

        for item in current_list:
            if isinstance(item, list):
                # still in a list of lists, recurse
                return ArrayUtil.first_item(item)
            else:
                return item

    @staticmethod
    def next_item(current_list, new_list=True, nesting_change=0,
                  end_of_list=True):
        # returns the next item in a nested list along with other information:
        # (<next item>, <end of list>, <entering new list>,
        #  <change in nesting level>
        if not isinstance(current_list, list) and \
                not isinstance(current_list, np.ndarray):
            yield (current_list, end_of_list, new_list, nesting_change)
        else:
            list_size = 1
            for item in current_list:
                if isinstance(item, list) or isinstance(current_list,
                                                        np.ndarray):
                    # still in a list of lists, recurse
                    for item in ArrayUtil.next_item(item, list_size == 1,
                                                    nesting_change + 1,
                                                    list_size ==
                                                    len(current_list)):
                        yield item
                    nesting_change = -(nesting_change + 1)
                else:
                    yield (item, list_size == len(current_list),
                           list_size == 1, nesting_change)
                    nesting_change = 0
                list_size += 1

    @staticmethod
    def next_list(current_list):
        if not isinstance(current_list[0], list):
            yield current_list
        else:
            for lst in current_list:
                if isinstance(lst[0], list):
                    for lst in ArrayUtil.next_list(lst):
                        yield lst
                else:
                    yield lst

    def array_comp(self, first_array, second_array):
        diff = first_array - second_array
        max = np.max(np.abs(diff))
        if max > self.max_error:
            return False
        return True

    @staticmethod
    def reset_delimiter_used():
        ArrayUtil.delimiter_used = None
        ArrayUtil.line_num = 0
        ArrayUtil.consistent_delim = True

    @staticmethod
    def split_data_line(line, external_file=False, delimiter_conf_length=15):
        if ArrayUtil.line_num > delimiter_conf_length and \
                ArrayUtil.consistent_delim:
            # consistent delimiter has been found.  continue using that
            # delimiter without doing further checks
            if ArrayUtil.delimiter_used == None:
                clean_line = line.strip().split()
            else:
                clean_line = line.strip().split(ArrayUtil.delimiter_used)
        else:
            clean_line = line.strip().split()
            if external_file:
                # try lots of different delimitiers for external files and use the
                # one the breaks the data apart the most
                max_split_size = len(clean_line)
                max_split_type = None
                for delimiter in ArrayUtil.delimiter_list:
                    alt_split = line.strip().split(delimiter)
                    if len(alt_split) > max_split_size:
                        max_split_size = len(alt_split)
                        max_split_type = delimiter
                if max_split_type is not None:
                    clean_line = line.strip().split(max_split_type)
                    if ArrayUtil.line_num == 0:
                        ArrayUtil.delimiter_used = max_split_type
                    elif ArrayUtil.delimiter_used != max_split_type:
                        ArrayUtil.consistent_delim = False
        ArrayUtil.line_num += 1

        arr_fixed_line = []
        index = 0
        # loop through line to fix quotes and delimiters
        while index < len(clean_line):
            item = clean_line[index]
            if item and item not in ArrayUtil.delimiter_list:
                if item and item[0] in ArrayUtil.quote_list:
                    # starts with a quote, handle quoted text
                    if item[-1] in ArrayUtil.quote_list:
                        arr_fixed_line.append(item[1:-1])
                    else:
                        arr_fixed_line.append(item[1:])
                        # loop until trailing quote found
                        while index < len(clean_line):
                            index += 1
                            if index < len(clean_line):
                                item = clean_line[index]
                                if item[-1] in ArrayUtil.quote_list:
                                    arr_fixed_line[-1] = \
                                        '{} {}'.format(arr_fixed_line[-1],
                                                       item[:-1])
                                    break
                                else:
                                    arr_fixed_line[-1] = \
                                        '{} {}'.format(arr_fixed_line[-1],
                                                       item)
                else:
                    # no quote, just append
                    arr_fixed_line.append(item)
            index += 1

        return arr_fixed_line

    @staticmethod
    def clean_numeric(text):
        if isinstance(text, str):
            # remove all non-numeric text from leading and trailing positions
            # of text
            if text:
                while text and (text[0] not in ArrayUtil.numeric_chars or text[-1]
                                not in ArrayUtil.numeric_chars):
                    if text[0] not in ArrayUtil.numeric_chars:
                        text = text[1:]
                    if text and text[-1] not in ArrayUtil.numeric_chars:
                        text = text[:-1]
        return text

    def save_array_diff(self, first_array, second_array, first_array_name,
                        second_array_name):
        try:
            diff = first_array - second_array
            self.save_array(first_array_name, first_array)
            self.save_array(second_array_name, second_array)
            self.save_array('debug_array_diff.txt', diff)
        except:
            print("An error occurred while outputting array differences.")
            return False
        return True

    # Saves an array with up to three dimensions
    def save_array(self, filename, multi_array):
        file_path = os.path.join(self.path, filename)
        with open(file_path, 'w') as outfile:
            outfile.write('{}\n'.format(str(multi_array.shape)))
            if len(multi_array.shape) == 4:
                for slice in multi_array:
                    for second_slice in slice:
                        for third_slice in second_slice:
                            for item in third_slice:
                                outfile.write(' {:10.3e}'.format(item))
                            outfile.write('\n')
                        outfile.write('\n')
                    outfile.write('\n')
            elif len(multi_array.shape) == 3:
                for slice in multi_array:
                    np.savetxt(outfile, slice, fmt='%10.3e')
                    outfile.write('\n')
            else:
                np.savetxt(outfile, multi_array, fmt='%10.3e')

class MultiList():
    """
    Class for storing objects in an n-dimensional list which can be iterated
    through as a single list.

    Parameters
    ----------
    mdlist : list
        multi-dimensional list to initialize the multi-list.  either mdlist
        or both shape and callback must be specified
    shape : tuple
        shape of the multi-list
    callback : method
        callback method that takes a location in the multi-list (tuple) and
        returns an object to be stored at that location in the multi-list

    Methods
    -------
    increment_dimension : (dimension, callback)
        increments the size of one of the two dimensions of the multi-list
    build_list : (callback)
        builds a multi-list of shape self.list_shape, constructing objects
        for the list using the supplied callback method
    first_item : () : object
        gets the first entry in the multi-list
    get_total_size : () : int
        returns the total number of entries in the multi-list
    in_shape : (indexes) : boolean
        returns whether a tuple of indexes are valid indexes for the shape of
        the multi-list
    inc_shape_idx : (indexes) : tuple
        given a tuple of indexes pointing to an entry in the multi-list,
        returns a tuple of indexes pointing to the next entry in the multi-list
    first_index : () : tuple
        returns a tuple of indexes pointing to the first entry in the
        multi-list
    indexes : (start_indexes=None, end_indexes=None) : iter(tuple)
        returns an iterator that iterates from the location in the
        multi-list defined by start_indexes to the location in the
        multi-list defined by end_indexes
    elements : () : iter(object)
        returns an iterator that iterates over each object stored in the
        multi-list
    """
    def __init__(self, mdlist=None, shape=None, callback=None):
        if mdlist is not None:
            self.multi_dim_list = mdlist
            self.list_shape = MultiList._calc_shape(mdlist)
        elif shape is not None:
            self.list_shape = shape
            self.multi_dim_list = []
            if callback is not None:
                self.build_list(callback)
        else:
            raise Exception('MultiList requires either a mdlist or a shape '
                            'at initialization.')

    def __getitem__(self, k):
        if isinstance(k, list) or isinstance(k, tuple):
            item_ptr = self.multi_dim_list
            for index in k:
                item_ptr = item_ptr[index]
            return item_ptr
        else:
            return self.multi_dim_list[k]

    @staticmethod
    def _calc_shape(current_list):
        shape = []
        if isinstance(current_list, list):
            shape.append(len(current_list))
            sub_list = current_list[0]
            if isinstance(sub_list, list):
                shape += MultiList._calc_shape(sub_list)
        elif isinstance(current_list, np.ndarray):
            shape.append(current_list.shape[0])
        else:
            return 1
        return tuple(shape)

    def increment_dimension(self, dimension, callback):
        # ONLY SUPPORTS 1 OR 2 DIMENSIONAL MULTI-LISTS
        # TODO: REWRITE TO SUPPORT N-DIMENSIONAL MULTI-LISTS
        if len(self.list_shape) > 2:
            raise Exception('Increment_dimension currently only supports 1 '
                            'or 2 dimensional multi-lists')
        if len(self.list_shape) == 1:
            self.multi_dim_list.append(callback(len(self.list_shape)))
            self.list_shape = (self.list_shape[0] + 1,)
        else:
            if dimension == 1:
                new_row_idx = len(self.multi_dim_list)
                self.multi_dim_list.append([])
                for index in range(0, self.list_shape[1]):
                    self.multi_dim_list[-1].append(callback((new_row_idx,
                                                             index)))
                self.list_shape = (self.list_shape[0] + 1, self.list_shape[1])
            elif dimension == 2:
                new_col_idx = len(self.multi_dim_list[0])
                for index in range(0, self.list_shape[0]):
                    self.multi_dim_list[index].append(callback((index,
                                                                new_col_idx)))
                self.list_shape = (self.list_shape[0], self.list_shape[1] + 1)
            else:
                raise Exception('For two dimensional lists "dimension" must '
                                'be 1 or 2.')

    def build_list(self, callback):
        entry_points = [(self.multi_dim_list, self.first_index())]
        shape_len = len(self.list_shape)
        # loop through each dimension
        for index, shape_size in enumerate(self.list_shape):
            new_entry_points = []
            # loop through locations to add to the list
            for entry_point in entry_points:
                # loop through the size of current dimension
                for val in range(0, shape_size):
                    if index < (shape_len - 1):
                        # this is a multi-dimensional multi-list, build out
                        # first dimension
                        entry_point[0].append([])
                        if entry_point[1] is None:
                            new_location = (len(entry_point) - 1,)
                        else:
                            new_location = ((len(entry_point[0]) - 1), val)
                        new_entry_points.append((entry_point[0][-1],
                                                 new_location))
                    else:
                        entry_point[0].append(callback(entry_point[1]))
            entry_points = new_entry_points

    def first_item(self):
        return ArrayUtil.first_item(self.multi_dim_list)

    def get_total_size(self):
        shape_size = 1
        for item in self.list_shape:
            if item is None:
                return 0
            else:
                shape_size *= item
        return shape_size

    def in_shape(self, indexes):
        for index, item in zip(indexes, self.list_shape):
            if index > item:
                return False
        return True

    def inc_shape_idx(self, indexes):
        new_indexes = []
        incremented = False
        for index, item in zip(indexes, self.list_shape):
            if index == item:
                new_indexes.append(0)
            elif incremented:
                new_indexes.append(index)
            else:
                incremented = True
                new_indexes.append(index+1)
        if not incremented:
            new_indexes[-1] += 1
        return tuple(new_indexes)

    def first_index(self):
        first_index = []
        for index in self.list_shape:
            first_index.append(0)
        return tuple(first_index)

    def nth_index(self, n):
        index = None
        aii = ArrayIndexIter(self.list_shape, True)
        index_num = 0
        while index_num <= n:
            index = aii.next()
            index_num += 1
        return index

    def indexes(self, start_indexes=None, end_indexes=None):
        aii = ArrayIndexIter(self.list_shape, True)
        if start_indexes is not None:
            aii.current_location = list(start_indexes)
            aii.current_index = len(aii.current_location) - 1
        if end_indexes is not None:
            aii.end_location = list(end_indexes)
        return aii

    def elements(self):
        return MultiListIter(self.multi_dim_list, False)

class ArrayIndexIter(object):
    def __init__(self, array_shape, index_as_tuple=False):
        self.array_shape = array_shape
        self.current_location = []
        self.end_location = []
        self.first_item = True
        self.index_as_tuple = index_as_tuple
        for item in array_shape:
            self.current_location.append(0)
            self.end_location.append(item)
        self.current_index = len(self.current_location) - 1

    def __iter__(self):
        return self

    def __next__(self):
        if self.first_item:
            self.first_item = False
            if self.current_location[self.current_index] < \
                    self.end_location[self.current_index]:
                if len(self.current_location) > 1 or self.index_as_tuple:
                    return tuple(self.current_location)
                else:
                    return self.current_location[0]
        while self.current_index >= 0:
            location = self.current_location[self.current_index]
            if location < self.end_location[self.current_index] - 1:
                self.current_location[self.current_index] += 1
                self.current_index = len(self.current_location) - 1
                if len(self.current_location) > 1 or self.index_as_tuple:
                    return tuple(self.current_location)
                else:
                    return self.current_location[0]
            else:
                self.current_location[self.current_index] = 0
                self.current_index -= 1
        raise StopIteration()

    next = __next__  # Python 2 support

class MultiListIter(object):
    def __init__(self, multi_list, detailed_info=False, iter_leaf_lists=False):
        self.multi_list = multi_list
        self.detailed_info = detailed_info
        if iter_leaf_lists:
            self.val_iter = ArrayUtil.next_list(self.multi_list)
        else:
            self.val_iter = ArrayUtil.next_item(self.multi_list)

    def __iter__(self):
        return self

    def __next__(self):
        next_val = next(self.val_iter)
        if self.detailed_info:
            return next_val
        else:
            return next_val[0]

    next = __next__  # Python 2 support

class ConstIter(object):
    def __init__(self, value):
        self.value = value

    def __iter__(self):
        return self

    def __next__(self):
        return self.value

    next = __next__  # Python 2 support

class FileIter(object):
    def __init__(self, file_path):
        self.eof = False
        try:
            self._fd = open(file_path, 'r')
        except:
            self.eof = True
        self._current_data = None
        self._data_index = 0
        self._next_line()

    def __iter__(self):
        return self

    def __next__(self):
        if self.eof:
            raise StopIteration()
        else:
            while self._current_data is not None and \
                  self._data_index >= len(self._current_data):
                self._next_line()
                self._data_index = 0
                if self.eof:
                    raise StopIteration()
            self._data_index += 1
            return self._current_data[self._data_index-1]

    def close(self):
        self._fd.close()

    def _next_line(self):
        if self.eof:
            return
        data_line = self._fd.readline()
        if data_line is None:
            self.eof = True
            return
        self._current_data = ArrayUtil.split_data_line(data_line)

    next = __next__  # Python 2 support

class NameIter(object):
    def __init__(self, name, first_not_numbered=True):
        self.name = name
        self.iter_num = -1
        self.first_not_numbered = first_not_numbered

    def __iter__(self):
        return self

    def __next__(self):
        self.iter_num += 1
        if self.iter_num == 0 and self.first_not_numbered:
            return self.name
        else:
            return '{}_{}'.format(self.name, self.iter_num)

    next = __next__  # Python 2 support

class PathIter(object):
    def __init__(self, path, first_not_numbered=True):
        self.path = path
        self.name_iter = NameIter(path[-1], first_not_numbered)

    def __iter__(self):
        return self

    def __next__(self):
        return self.path[0:-1] + (self.name_iter.__next__(),)

    next = __next__  # Python 2 support

class MFDocString(object):
    """
    Helps build a python class doc string

    Parameters
    ----------
    description : string
        description of the class

    Attributes
    ----------
    indent: string
        indent to use in doc string
    description : string
        description of the class
    parameter_header : string
        header for parameter section of doc string
    parameters : list
        list of docstrings for class parameters

    Methods
    -------
    add_parameter : (param_descr : string, beginning_of_list : bool)
        adds doc string for a parameter with description 'param_descr' to the
        end of the list unless beginning_of_list is True
    get_doc_string : () : string
        builds and returns the docstring for the class
    """
    def __init__(self, description):
        self.indent = '    '
        self.description = description
        self.parameter_header = '{}Parameters\n{}' \
                                '----------'.format(self.indent, self.indent)
        self.parameters = []
        self.model_parameters = []

    def add_parameter(self, param_descr, beginning_of_list=False,
                      model_parameter=False):
        if beginning_of_list:
            self.parameters.insert(0, param_descr)
            if model_parameter:
                self.model_parameters.insert(0, param_descr)
        else:
            self.parameters.append(param_descr)
            if model_parameter:
                self.model_parameters.append(param_descr)

    def get_doc_string(self, model_doc_string=False):
        doc_string = '{}"""\n{}{}\n\n{}\n'.format(self.indent, self.indent,
                                                  self.description,
                                                  self.parameter_header)
        if model_doc_string:
            param_list = self.model_parameters
            doc_string = '{}    modelname : string\n        name of the ' \
                         'model\n    model_nam_file : string\n' \
                         '        relative path to the model name file from ' \
                         'model working folder\n    version : string\n' \
                         '        version of modflow\n    exe_name : string\n'\
                         '        model executable name\n' \
                         '    model_ws : string\n' \
                         '        model working folder path' \
                         '\n'.format(doc_string)
        else:
            param_list = self.parameters
        for parameter in param_list:
            doc_string += '{}\n'.format(parameter)
        if not model_doc_string:
            doc_string += '\n{}"""'.format(self.indent)
        return doc_string


import sys
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import Optional, Sequence, Text

from ezno_convert.common import VERSION, DATE_FORMAT
from ezno_convert.convert import WORDConverter, PPTConverter, XLConverter
from ezno_convert.enums import WORD, PPT, XL

class CommandLineInterface(ArgumentParser):
    def __init__(self):
        super().__init__(description=f'Native Office Converter {VERSION}')
        self.add_argument('PATH', nargs='+', type=Path, help='''
        Path(s) of files or folders to convert.
        If a folder is specified, include all files in it (to filter types, see folder options below)
        ''')
        self.add_argument('-o', '--output', type=Path, metavar='PATH', required=False, help='''
        Where to save output files.
        If input PATH is a folder (or multiple paths) this output PATH must also be a folder.
        Default is to save output files in the same folder as input files.
        If the output PATH isn't a folder (input is a file) and the proper extension is omitted,
        it will be added automatically.
        ''')
        self.add_argument('-c', '--converter', metavar='TYPE', default='PDF', help='''
        Type of conversion to preform. For available types see --list_types (Default: %(default)s)
        ''')
        self.add_argument('-t', '--no_timestamp', action='store_true', help='''
        Do not add timestamp to output filenames. Warning: this will overwrite existing output files.
        ''')
        self.add_argument('-d', '--dateformat', default=DATE_FORMAT, help='''
        Specify datetime format to add to filenames. Ignored if --no-timestamp specified. Default: %(default)s
        ''')
        # self.add_argument('-s', '--simulate', action='store_true',
        #                   help='List files and simulate conversions without actually converting anything')
        self.add_argument('-l', '--list_types', action='store_true', help='Print available conversion types and exit')
        self.add_argument('-v', '--version', action='version', version=f'%(prog)s {VERSION}')

        folder = self.add_argument_group(
            title='Folder Options',
            description='These options apply only to input paths that are folders, they are ignored otherwise'
        )
        folder.add_argument('-r', '--recursive', action='store_true', help='Search in sub-folders recursively')
        folder.add_argument('-w', '--word', action='store_true',
                            help=f'Convert all Word Documents {WORD.extensions.value}')
        folder.add_argument('-p', '--powerpoint', action='store_true',
                            help=f'Convert all PowerPoint Presentations {PPT.extensions.value}')
        folder.add_argument('-x', '--excel', action='store_true',
                            help=f'Convert all Excel Spreadsheets {XL.extensions.value}')
        folder.add_argument('-a', '--all', action='store_true', help=f'''
        Convert all possible files {WORD.extensions.value + PPT.extensions.value + XL.extensions.value}.
        Possible conversions depend on conversion type.
        Example 1: "-a -c PDF" and "-a -c XPS" will apply to all file types
        Example 2: "-a -c AnimatedGIF" will only apply to PowerPoint Presentations even without "-p"
        ''')

        xl = self.add_argument_group(
            title='Excel only options',
            description=f'These options apply only to excel files {XL.extensions.value}, they are ignored otherwise'
        )
        xl.add_argument('--split', action='store_true', help='Convert each worksheet separately')
        xl.add_argument('--sheet', nargs='+', help='''
        Specify names or indexes of specific sheets to convert, instead of converting the entire file. Implies --split
        ''')

    def parse_args(self, args: Optional[Sequence[Text]] = None) -> Namespace:
        args = super().parse_args(args)
        dirs = [p for p in args.PATH if p.is_dir()]

        for p in args.PATH:
            if not p.is_file() and not p.is_dir():
                print(f'Path not valid: {p}', file=sys.stderr)
        args.PATH = [p for p in args.PATH if p.is_file() or p.is_dir()]
        if not args.PATH:
            self.error('No paths were valid')

        if len(args.PATH) == 1 and args.PATH[0].is_file():
            valid_output = args.output is None or args.output.is_dir() or args.output.parent.is_dir()
        elif len(args.PATH) > 1 or dirs:
            valid_output = args.output is None or args.output.is_dir()
        else:
            valid_output = False

        if not valid_output:
            self.error(f'Output path invalid ({args.output})')

        if not any((args.word, args.powerpoint, args.excel)):
            args.all = True

        if args.no_timestamp:
            args.dateformat = None

        word = [p for p in args.PATH if p.suffix in WORD.extensions.value]
        args.word = word + dirs if (args.word or args.all) and hasattr(WORD, args.converter) else []
        excel = [p for p in args.PATH if p.suffix in XL.extensions.value]
        args.excel = excel + dirs if (args.excel or args.all) and hasattr(XL, args.converter) else []
        powerpoint = [p for p in args.PATH if p.suffix in PPT.extensions.value]
        args.powerpoint = powerpoint + dirs if (args.powerpoint or args.all) and hasattr(PPT, args.converter) else []

        if args.sheet:
            args.sheet = [int(sh) if sh.isdigit() else sh for sh in args.sheet]
        elif args.split:
            args.sheet = True

        return args

    def run_converters(self, args: Optional[Sequence[Text]] = None):
        opt = self.parse_args(args)
        kwargs = dict(dst=opt.output, recursive=opt.recursive, date_fmt=opt.dateformat)

        if opt.word:
            word_gen = WORDConverter(src=opt.word, target=getattr(WORD, opt.converter, None), **kwargs)
            word_gen.execute_all(True)
        if opt.powerpoint:
            pp_gen = PPTConverter(src=opt.powerpoint, target=getattr(PPT, opt.converter, None), **kwargs)
            pp_gen.execute_all(True)
        if opt.excel:
            xl_gen = XLConverter(src=opt.excel, target=getattr(XL, opt.converter, None), sheets=opt.sheet, **kwargs)
            xl_gen.execute_all(True)

def main():
    CommandLineInterface().run_converters()

if __name__ == '__main__':
    main()