import argparse
import subprocess

from itertools import chain
from maas_common import metric
from maas_common import metric_bool
from maas_common import print_output
from maas_common import status_err, status_err_no_exit
from maas_common import status_ok
import re
import requests

OVERVIEW_URL = "%s://%s:%s/api/overview"
NODES_URL = "%s://%s:%s/api/nodes"
CONNECTIONS_URL = "%s://%s:%s/api/connections?columns=channels"
QUEUES_URL = "%s://%s:%s/api/queues"
CONSUMERS_QUEUES_URL = "%s://%s:%s/api/queues/%s"
VHOSTS_URL = "%s://%s:%s/api/vhosts"

CLUSTERED = True
# NOTE(cloudnull): The cluster size is set using a Jinja2 variable
#                  to get this usage to pass flake8 the jinja is
#                  set as a string and then converted to an int.
_CLUSTER_SIZE = """{{ groups['rabbitmq_all'] | default([]) | length }}"""
CLUSTER_SIZE = int(_CLUSTER_SIZE)

# {metric_category: {metric_name: metric_unit}}
OVERVIEW_METRICS = {"queue_totals": {"messages": "messages",
                                     "messages_ready": "messages",
                                     "messages_unacknowledged": "messages"},
                    "message_stats": {"get": "messages",
                                      "ack": "messages",
                                      "deliver_get": "messages",
                                      "deliver": "messages",
                                      "publish": "messages"}}
# {metric_name: metric_unit}
NODES_METRICS = {"proc_used": "processes",
                 "proc_total": "processes",
                 "fd_used": "fd",
                 "fd_total": "fd",
                 "sockets_used": "fd",
                 "sockets_total": "fd",
                 "mem_used": "bytes",
                 "mem_limit": "bytes",
                 "mem_alarm": "status",
                 "disk_free_alarm": "status",
                 "uptime": "ms"}

CONNECTIONS_METRICS = {"max_channels_per_conn": "channels"}

def hostname():
    """Return the name of the current host/node."""
    return subprocess.check_output(['hostname', '-s']).strip()

def rabbit_version(node):
    if 'applications' in node:
        if 'rabbit' in node['applications']:
            if 'version' in node['applications']['rabbit']:
                version_string = node['applications']['rabbit']['version']
                return tuple(int(part) for part in version_string.split('.'))
    else:
        return tuple()

def parse_args():
    parser = argparse.ArgumentParser(description='RabbitMQ checks')
    parser.add_argument('-H', '--host', action='store', dest='host',
                        default='localhost',
                        help='Host address to use when connecting')
    parser.add_argument('-P', '--port', action='store', dest='port',
                        default='15672',
                        help='Port to use when connecting')
    parser.add_argument('-U', '--username', action='store', dest='username',
                        default='guest',
                        help='Username to use for authentication')
    parser.add_argument('-p', '--password', action='store', dest='password',
                        default='guest',
                        help='Password to use for authentication')
    parser.add_argument('-n', '--name', action='store', dest='name',
                        default=None,
                        help=("Check a node's cluster membership using the "
                              'provided name'))
    parser.add_argument('--telegraf-output',
                        action='store_true',
                        default=False,
                        help='Set the output format to telegraf')
    parser.add_argument('--http',
                        action='store_true',
                        help='Use http for checks')
    parser.add_argument('--https',
                        action='store_true',
                        help='Use https for checks')
    return parser.parse_args()

def _get_rabbit_json(session, url):
    try:
        response = session.get(url, verify=False)
    except requests.exceptions.ConnectionError as e:
        metric_bool('client_success', False, m_name='maas_rabbitmq')
        status_err(str(e), m_name='maas_rabbitmq')

    if response.ok:
        return response.json()
    else:
        metric_bool('client_success', False, m_name='maas_rabbitmq')
        status_err('Received status {0} from RabbitMQ API'.format(
            response.status_code), m_name='maas_rabbitmq')

def _get_connection_metrics(session, metrics, protocol, host, port):

    response = _get_rabbit_json(session, CONNECTIONS_URL % (protocol,
                                                            host, port))

    max_chans = max(chain(connection['channels'] for connection in response
                    if 'channels' in connection))
    for k in CONNECTIONS_METRICS:
        metrics[k] = {'value': max_chans, 'unit': CONNECTIONS_METRICS[k]}

def _get_overview_metrics(session, metrics, protocol, host, port):
    response = _get_rabbit_json(session, OVERVIEW_URL % (protocol, host, port))

    for k in OVERVIEW_METRICS:
        if k in response:
            for a, b in OVERVIEW_METRICS[k].items():
                if a in response[k]:
                    metrics[a] = {'value': response[k][a], 'unit': b}

def _get_node_metrics(session, metrics, protocol, host, port, name):
    response = _get_rabbit_json(session, NODES_URL % (protocol, host, port))

    # Either use the option provided by the commandline flag or the current
    # hostname
    name = '@' + (name or hostname())
    is_cluster_member = False

    # Ensure this node is a member of the cluster
    nodes_matching_name = [n for n in response
                           if n['name'].endswith(name)]
    is_cluster_member = any(nodes_matching_name)

    if CLUSTERED:
        if len(response) < CLUSTER_SIZE:
            status_err_no_exit('cluster too small', m_name='maas_rabbitmq')
        if not is_cluster_member:
            status_err_no_exit('{0} not a member of the cluster'.format(name),
                               m_name='maas_rabbitmq')
        if sum([len(n.get('partitions', [])) for n in response]):
            status_err_no_exit('At least one partition found in the rabbit '
                               'cluster', m_name='maas_rabbitmq')
        if any([len(n.get('cluster_links', [])) != CLUSTER_SIZE - 1
                for n in response]):
            status_err_no_exit('At least one rabbit node is missing a cluster'
                               ' link', m_name='maas_rabbitmq')

    for k, v in NODES_METRICS.items():
        metrics[k] = {'value': nodes_matching_name[0][k], 'unit': v}

def _get_queue_metrics(session, metrics, protocol, host, port):
    response = _get_rabbit_json(session, QUEUES_URL % (protocol, host, port))
    msgs_excl_notifications = 0
    notification_messages = 0
    msgs_without_consumers = 0
    for queue in response:
        messages = queue.get('messages')
        consumers = queue.get('consumers')
        name = queue.get('name')
        # RabbitMQ sometimes responds w/o messages or consumers (TURTLES-715)
        if messages is None or consumers is None or name is None:
            continue
        if consumers > 0:
            if re.match(r'^/?(versioned_)?notifications\.', name):
                notification_messages += messages
            else:
                msgs_excl_notifications += messages
        elif consumers == 0:
            msgs_without_consumers += messages

    metrics['notification_messages'] = {
        'value': notification_messages,
        'unit': 'messages'
    }
    metrics['msgs_excl_notifications'] = {
        'value': msgs_excl_notifications,
        'unit': 'messages'
    }
    metrics['msgs_without_consumers'] = {
        'value': msgs_without_consumers,
        'unit': 'messages'
    }

def _get_consumer_metrics(session, metrics, protocol, host, port):
    VHOSTS = []
    vhosts_response = _get_rabbit_json(session, VHOSTS_URL % (protocol,
                                                              host, port))
    # get the vhosts
    for vhost in vhosts_response:
        VHOSTS.append(vhost['name'])

    queues_without_consumers = 0
    for vhost in VHOSTS:
        # uri encode /
        URI_ENCODED = vhost.replace('/', '%2F')
        queue_response = _get_rabbit_json(session, CONSUMERS_QUEUES_URL %
                                          (protocol, host, port, URI_ENCODED))
        for queue in queue_response:
            if queue.get('consumers', 0) == 0 and queue.get('messages', 0) > 0:
                queues_without_consumers += 1

    metrics['queues_without_consumers'] = {
        'value': queues_without_consumers,
        'unit': 'queues'
    }

def main():
    metrics = {}
    session = requests.Session()  # Make a Session to store the auth creds
    session.auth = (options.username, options.password)

    protocol = 'https' if options.https else 'http'

    _get_connection_metrics(session, metrics, protocol,
                            options.host, options.port)
    _get_overview_metrics(session, metrics, protocol,
                          options.host, options.port)
    _get_node_metrics(session, metrics, protocol, options.host,
                      options.port, options.name)
    _get_queue_metrics(session, metrics, protocol, options.host,
                       options.port)
    _get_consumer_metrics(session, metrics, protocol, options.host,
                          options.port)

    status_ok(m_name='maas_rabbitmq')

    for k, v in metrics.items():
        if v['value'] is True or v['value'] is False:
            metric_bool('rabbitmq_%s_status' % k, not v['value'])
        else:
            metric('rabbitmq_%s' % k, 'int64', v['value'], v['unit'])

if __name__ == "__main__":
    args = options = parse_args()
    with print_output(print_telegraf=args.telegraf_output):
        main()


# scraping
import requests, json, lxml
from bs4 import BeautifulSoup
from celery import shared_task
from datetime import datetime
from .models import News

@shared_task(serializer='json')
def save_function(article_list):
    print('starting save items')
    news_count = 0

    for article in article_list:
        try:
            News.objects.create(
                title = article['title'],
                link = article['link'],
                published = article['published'],
                source = article['source']
            )
            news_count += 1
        except Exception as e:
            print('failed at latest_article is none')
            print(e)
            break
    return print('finished')

# scraping function
@shared_task(name='news_scraping')
def hackernews_rss():
    article_list = []
    try:
        print('Starting the scraping tool')
        # execute my request, parse the data using XML
        # parser in BS4
        r = requests.get('https://news.ycombinator.com/rss')
        soup = BeautifulSoup(r.content, features='xml')
        # select only the "items" I want from the data
        articles = soup.findAll('item')

        # for each "item" I want, parse it into a list
        for a in articles:
            title = a.find('title').text
            link = a.find('link').text
            published_wrong = a.find('pubDate').text
            published = datetime.strptime(published_wrong, '%a, %d %b %Y %H:%M:%S %z')
            # print(published, published_wrong) # checking correct date format
            # create an "article" object with the data
            # from each "item"
            article = {
                'title': title,
                'link': link,
                'published': published,
                'source': 'HackerNews RSS'
            }
            # append my "article_list" with each "article" object
            article_list.append(article)
            print('Finished scraping the articles')

            # after the loop, dump my saved objects into a .txt file
            return save_function(article_list)
    except Exception as e:
        print('The scraping job failed. See exception:')
        print(e)


import json
import os
import numpy as np

from astropy.io import ascii
from astropy.io import fits as pyfits
from astropy.io.misc import yaml
from astropy.io import fits as pyfits
from astropy.coordinates import SkyCoord
from astropy.table import Table
from astropy import units as u

from ext2TeV import PACKAGE_DATA
from ext2TeV.utils import read_yaml

class Catalog(object):
    def reset_spectral_points(self):
        self.xval = []
        self.xerrn = []
        self.xerrp = []
        self.yval = []
        self.yerrn = []
        self.yerrp = []
        #self.residuals = Residuals()
        #self.spectra   = Spectra()

    #@staticmethod
    def match_src_catalog(self, catalog, src, tol=0.5*u.Unit("deg")):
        coord_catalog = SkyCoord(
            ra=catalog['ra'] * u.Unit("deg"),
            dec=catalog['dec'] * u.Unit("deg")
        )

        c = SkyCoord(
            ra=src['ra'] * u.Unit("deg"),
            dec=src['dec'] * u.Unit("deg")
        )

        idx, d2d, d3d = c.match_to_catalog_3d(coord_catalog)
        #print(idx, d2d, d3d)
        if d2d > tol:
            return None

        return catalog['id'][idx]

    def reset_results(self):
        self.result = dict()

class GammaCAT(Catalog):
    def read(self,f):
        with open(f) as temp:
            self.contents = yaml.load(temp.read())
            #return(self.contents)

    def build(self,f,ind,outf="gammacat_export.yaml"):

        os.chdir(ind)
        with open("gammacat.yaml") as gammacat:
            temp = yaml.load(gammacat.read())
            gammacat = dict()
            for k,source_id in enumerate(temp):
                gammacat[source_id[0][1]] = dict()
                for pair in temp[k]:
                    gammacat[source_id[0][1]][pair[0]] = pair[1]

        with open("gammacat-datasets.json") as datasets:
            temp = json.load(datasets)
            datasets = dict()
            for item in temp:
                if item['source_id'] not in datasets:
                    datasets[item['source_id']] = dict()

                # copy identifiers from gammacat
                if item['source_id'] in gammacat:
                    print('Appending data from gammacat[{0}]'.format(item['source_id']))
                    for tocopy in gammacat[item['source_id']]:
                        datasets[item['source_id']][tocopy] = gammacat[item['source_id']][tocopy]
                else:
                    print('Fallback, getting the info from the sources dir')
                    info = read_yaml("sources/tev-{0:06d}.yaml".format(item['source_id']))
                    for tocopy in info:
                        datasets[item['source_id']][tocopy] = info[tocopy]

                if item['type'] == 'sed':
                    try:
                        content = open(item['location']).read() #Table.read(item['location'])
                    except:
                        print('Skipping {0}'.format(item['location']))

                    if 'sed' in datasets[item['source_id']]:
                        datasets[item['source_id']]['sed'].append(content)
                    else:
                        datasets[item['source_id']]['sed'] = [content]

        with open("gammacat_export_20210215.yaml","w+") as gcf:
            yaml.dump(datasets,gcf)

        self.contents = datasets

    def match_src(self,src,tol=0.5*u.Unit("deg")):
        list_items = self.contents.keys()
        catalog = {}
        catalog['dec'] = [self.contents[item]['dec'] \
                          for item in self.contents \
                          if 'dec' in self.contents[item]]
        catalog['ra']  = [self.contents[item]['ra'] \
                          for item in self.contents \
                          if 'dec' in self.contents[item]]
        catalog['id']  = [item for item in self.contents \
                          if 'dec' in self.contents[item]]

        try:
            self.matched = self.contents[self.match_src_catalog(catalog,src,tol)]
        except KeyError:
            self.matched = None
            return

    def get_spectralpoints(self):
        self.results = []
        for k, sed in enumerate(self.matched['sed']):
            energyerr = None
            # replace ECSV dump by its Table version
            sed = Table.read(sed, format='ascii.ecsv')
            #print(sed)
            energies = sed['e_ref'].to("TeV")
            flux = sed['dnde']
            if 'dnde_err' in sed.keys():
                fluxerr = sed['dnde_err']
                logFlux = 2 * np.log10(flux) - np.log10(flux + fluxerr)
                fluxerr = [flux - (10**(logFlux)), fluxerr]
            elif 'dnde_errp' in sed.keys():
                fluxerr = [sed['dnde_errn'], sed['dnde_errp']]
            else:
                print('No dnde_err*, check' )
                print(sed)

            isnotnan = ~np.isnan(flux) * (flux > 0)
            energies = energies[isnotnan]
            flux = flux[isnotnan]
            fluxerr = np.asarray([fluxerr[0][isnotnan], fluxerr[1][isnotnan]])

            # add units
            fluxerr = fluxerr*flux.unit

            if 'e_min' in sed.keys() and 'e_max' in sed.keys():
                energyerrn = (energies - sed['e_min'][isnotnan]).to(energies.unit)
                energyerrp = (sed['e_max'][isnotnan] - energies).to(energies.unit)

            if energyerr is None:
                energylog = np.log10(energies.value)
                energylogsep = np.min(energylog[1:] - energylog[:-1])
                energyerrn = energies.value - (10 ** (energylog - energylogsep / 2.))
                energyerrp = 10 ** (energylog + energylogsep / 2.) - energies.value

            e2flux = energies ** 2 * flux
            e2fluxerr = energies ** 2 * fluxerr

            result = {}
            result['name'] = self.matched['common_name']
            result['set'] = k
            result['E'] = energies.to("TeV")
            result['Eerr'] = np.asarray([energyerrn, energyerrp]) * energies.unit
            result['Eerr'] = result['Eerr'].to("TeV")
            result['EF'] = e2flux.to("erg/(cm2*s)")
            result['EFerr'] = e2fluxerr.to("erg/(cm2*s)")
            self.results.append(result)

class VTSCAT(Catalog):
    def read(self,f):
        with open(f) as temp:
            self.contents = yaml.load(temp.read())
            #return(self.contents)

    def build(self,ind,outf="vtscat_export.yaml"):
        #os.chdir(ind)
        vts_sources_yamls = sorted(glob.glob(ind+"/sources_vtscat/tev-*.yaml"))
        VTS_srcs = {}
        for fn in vts_sources_yamls:
            with open(fn) as f:
                YAMLcontents = yaml.load(f.read())
                VTS_srcs[YAMLcontents['source_id']]=YAMLcontents

        for indx in VTS_srcs:
            VTS_srcs[indx]['seds'] = []
            VTS_srcs[indx]['lcs'] = []

        list_of_papers = sorted(glob.glob(ind+"/20*/*"))
        list_of_seds = sorted(glob.glob(ind+"/20*/*/*-sed*.ecsv"))
        list_of_lcs = sorted(glob.glob(ind+"/20*/*/*-lc*.ecsv"))

        for sedfile in list_of_seds:
            sed = Table.read(sedfile, format='ascii.ecsv')
            sed_raw = open(sedfile).read()
            if sed.meta['source_id'] not in VTS_egal_srcs:
                continue

            src_dict = VTS_srcs[sed.meta['source_id']]
            src_dict['seds'].append(sed_raw)

        for lcfile in list_of_lcs:
            lc = Table.read(lcfile, format='ascii.ecsv')
            lc_raw = open(lcfile).read()
            if lc.meta['source_id'] not in VTS_srcs:
                continue

            src_dict = VTS_srcs[sed.meta['source_id']]
            src_dict['lcs'].append(lc_raw)

        with open("vtscat_export_20210215.yaml","w+") as vcf:
            yaml.dump(VTS_srcs,vcf)

        self.contents = VTS_srcs

    def reset_results(self):
        self.result = dict()

    def match_src(self,src,tol=0.5*u.Unit("deg")):
        list_items = self.contents.keys()
        catalog = {}

        try:
            src['ra']
        except KeyError:
            src['ra'] = src['pos']['ra']
            src['dec'] = src['pos']['dec']

        catalog['dec'] = [self.contents[item]['pos']['dec'] \
                          for item in self.contents \
                          if 'pos' in self.contents[item]]
        catalog['ra']  = [self.contents[item]['pos']['ra'] \
                          for item in self.contents \
                          if 'pos' in self.contents[item]]
        catalog['id']  = [self.contents[item]['source_id'] \
                          for item in self.contents \
                          if 'pos' in self.contents[item]]

        try:
            self.matched = self.contents[self.match_src_catalog(catalog,src,tol)]
        except KeyError:
            self.matched = None
            return

        if 'seds' not in self.matched:
            self.matched['seds']=[]
        if 'lcs' not in self.matched:
            self.matched['lcs']=[]

    def get_spectralpoints(self):
        self.results = []

        for k, sed in enumerate(self.matched['seds']):
            energyerr = None
            # replace ECSV dump by its Table version
            sed = Table.read(sed, format='ascii.ecsv')

            if 'dnde' in sed.colnames:
                flag_vts = np.isfinite(sed['dnde'])
            elif 'e2dnde' in sed.colnames:
                flag_vts = np.isfinite(sed['e2dnde'])

            if 'observatory' in sed.colnames:
                flag_vts *= sed['observatory']=='VERITAS'

            sed = sed[flag_vts]

            # Check if spectrum is valid, otherwise skip
            if len(sed)<=0:
                continue

            energies = sed['e_ref'].to("TeV")

            if 'dnde' in sed.colnames:
                flux = sed['dnde']
                if 'dnde_err' in sed.keys():
                    fluxerr = sed['dnde_err']
                    logFlux = 2*np.log10(flux/fluxerr.unit)-np.log10((flux/fluxerr.unit + fluxerr/fluxerr.unit))
                    fluxerr = [flux - (10**(logFlux))*fluxerr.unit, fluxerr]
                elif 'dnde_errp' in sed.keys():
                    fluxerr = [sed['dnde_errn'], sed['dnde_errp']]
                else:
                    print('No dnde_err*, check' )
            elif 'e2dnde' in sed.colnames:
                flux = sed['e2dnde']/(energies**2)
                if 'e2dnde_err' in sed.keys():
                    fluxerr = sed['e2dnde_err']/(energies.to("TeV")**2)
                    logFlux = 2*np.log10(flux/fluxerr.unit) - np.log10((flux/fluxerr.unit + fluxerr/fluxerr.unit))
                    fluxerr = [flux - (10**(logFlux))*fluxerr.unit, fluxerr]
                elif 'e2dnde_errp' in sed.keys():
                    fluxerr = [sed['e2dnde_errn']/(energies.to("TeV")**2),
                               sed['e2dnde_errp']/(energies.to("TeV")**2)]
                else:
                    print('No e2dnde_err*, check' )
            else:
                print('unexpected error happened, SED does not seem to have a dnde or e2dnde column ... ')
                continue

            isnotnan = ~np.isnan(flux) * (flux > 0)
            energies = energies[isnotnan]
            flux = flux[isnotnan]
            fluxerr = np.asarray([fluxerr[0][isnotnan], fluxerr[1][isnotnan]])

            # add units
            fluxerr = fluxerr*flux.unit

            if 'e_min' in sed.keys() and 'e_max' in sed.keys():
                energyerrn = (energies - sed['e_min'][isnotnan]).to(energies.unit)
                energyerrp = (sed['e_max'][isnotnan] - energies).to(energies.unit)

            if energyerr is None:
                energylog = np.log10(energies.value)
                energylogsep = np.min(energylog[1:] - energylog[:-1])
                energyerrn = energies.value - (10 ** (energylog - energylogsep / 2.))
                energyerrp = 10 ** (energylog + energylogsep / 2.) - energies.value

            e2flux = energies ** 2 * flux
            e2fluxerr = energies ** 2 * fluxerr

            result = {}
            result['name'] = self.matched['common_name']
            result['set'] = k
            result['E'] = energies.to("TeV")
            result['Eerr'] = np.asarray([energyerrn, energyerrp]) * energies.unit
            result['Eerr'] = result['Eerr'].to("TeV")
            result['EF'] = e2flux.to("erg/(cm2*s)")
            result['EFerr'] = e2fluxerr.to("erg/(cm2*s)")
            self.results.append(result)

class FermiCAT(Catalog):
    def __init__(self):
        pass

    def set_catalog_urls(self):
        baseurl = "https://fermi.gsfc.nasa.gov/ssc/data/access/lat"
        self.fhl3 = pyfits.open(baseurl+"/3FHL/gll_psch_v13.fit")
        self.fgl4 = pyfits.open(baseurl+"/10yr_catalog/gll_psc_v26.fit")
        self.lac4 = pyfits.open(baseurl+"/4LACDR2/table-4LAC-DR2-h.fits")
        self.lac4_low = pyfits.open(baseurl+"/4LACDR2/table-4LAC-DR2-l.fits")

    def set_active_catalog(self,cat):
        self.cat_name = cat
        if cat=='4LAC':
            self.get_spectralpoints_4fgl()
            import numpy.lib.recfunctions
            concat = numpy.lib.recfunctions.stack_arrays(\
                (lac4[1].data,lac4_low[1].data),
                autoconvert=True, usemask=False)

            self.cat = concat
        elif cat=='4FGL':
            self.get_spectralpoints_4fgl()
            self.cat = self.fgl4[1].data
        elif cat=='3FHL':
            self.get_spectralpoints_3fhl()
            self.cat = self.fhl3[1].data

    def get_spectralpoints_3fhl(self):
        self.raw_energies = self.fhl3[4].data

    def get_spectralpoints_4fgl(self):
        self.raw_energies = np.asarray([50, 100, 300, 1e3, 3e3, 1e4, 3e4, 3e5])*1e-3
        self.raw_energies = np.transpose([self.raw_energies[:-1],
                                          self.raw_energies[1:],
                                          0*self.raw_energies[1:]])

    def match_src(self,src,tol=0.5*u.Unit("deg")):
        catalog = dict()
        catalog['dec'] = self.cat['DEJ2000']
        catalog['ra']  = self.cat['RAJ2000']
        catalog['id']  = np.arange(len(catalog['ra']))
        self.matched = self.cat[self.match_src_catalog(catalog,src,tol)]
        if '4FGL' in self.matched['Source_Name']:
            self.lac4_from_fgl4()

    def extract_model_parameters(self):
        # Get model parameters from 4FGL/4LAC:
        self.result['pwl'] = dict()
        self.result['lp'] = dict()
        self.result['plec'] = dict()
        # Powerlaw
        self.result['pwl']['PivotE'] = self.matched['Pivot_Energy']*u.Unit("MeV")
        self.result['pwl']['Fnorm'] = self.matched['PL_Flux_Density']*u.Unit("MeV^-1/(s*cm2)")
        self.result['pwl']['EFnorm'] = self.matched['Unc_PL_Flux_Density']*u.Unit("MeV^-1/(s*cm2)")
        self.result['pwl']['Gamma'] = self.matched['PL_Index']
        self.result['pwl']['EGamma'] = self.matched['Unc_PL_Index']
        # LogParabola
        self.result['lp']['PivotE'] = self.matched['Pivot_Energy']*u.Unit("MeV")
        self.result['lp']['Fnorm'] = self.matched['LP_Flux_Density']*u.Unit("MeV^-1/(s*cm2)")
        self.result['lp']['EFnorm'] = self.matched['Unc_LP_Flux_Density']*u.Unit("MeV^-1/(s*cm2)")
        self.result['lp']['Gamma'] = self.matched['LP_Index']
        self.result['lp']['EGamma'] = self.matched['Unc_LP_Index']
        self.result['lp']['Beta'] = self.matched['LP_beta']
        self.result['lp']['EBeta'] = self.matched['Unc_LP_beta']
        # Powerlaw with exponential cutoff
        self.result['plec']['PivotE'] = self.matched['Pivot_Energy']*u.Unit("MeV")
        self.result['plec']['Fnorm'] = self.matched['PLEC_Flux_Density']*u.Unit("MeV^-1/(s*cm2)")
        self.result['plec']['EFnorm'] = self.matched['Unc_PLEC_Flux_Density']*u.Unit("MeV^-1/(s*cm2)")
        self.result['plec']['Gamma'] = self.matched['PLEC_Index']
        self.result['plec']['EGamma'] = self.matched['Unc_PLEC_Index']
        self.result['plec']['ExpF'] = self.matched['PLEC_Expfactor']
        self.result['plec']['EExpF'] = self.matched['Unc_PLEC_Expfactor']
        self.result['plec']['ExpI'] = self.matched['PLEC_Exp_Index']
        self.result['plec']['EExpI'] = self.matched['Unc_PLEC_Exp_Index']
        # Preferred model (4FGL)
        if self.matched['SpectrumType']=="PowerLaw":
            self.result['spectrumtype'] = "pwl"
        elif self.matched['SpectrumType']=="LogParabola":
            self.result['spectrumtype'] = "lp"
        elif self.matched['SpectrumType']=="PLExpCutoff":
            self.result['spectrumtype'] = "plec"

    def fill_spectrum(self):
        Eraw = self.raw_energies
        E = np.asarray([10 ** np.mean(np.log10(k[:-1])) for k in Eraw])
        Eerrn = np.asarray([E[k] - re[0] for k, re in enumerate(Eraw)])
        Eerrp = np.asarray([re[1] - E[k] for k, re in enumerate(Eraw)])

        # Add units
        F      = self.matched['Flux_Band']* u.Unit("cm**-2 * s**-1")
        Ferr   = np.transpose(np.abs(self.matched['Unc_Flux_Band']))*u.Unit("cm**-2 * s**-1")
        E      = E * u.Unit("GeV")
        SED    = (F * E).to("erg/(cm2*s)")
        SEDerr = (Ferr * E).to("erg/(cm2*s)")

        isnotnan = (~np.isnan(F))*\
                   (~np.isnan(Ferr[0])) * (F > 0)

        self.result['E'] = E[isnotnan].to("TeV")
        Eerr = np.asarray([Eerrn[isnotnan],Eerrp[isnotnan]])*u.Unit("GeV")
        self.result['Eerr'] = Eerr.to("TeV")
        self.result['EF'] = SED[isnotnan]
        self.result['EFerr'] = np.asarray(
            [SEDerr[0][isnotnan],SEDerr[1][isnotnan]])
        self.result['EFerr'] = self.result['EFerr']*SED.unit
        #return(self.result)

    def lac4_from_fgl4(self):
        srcnamefilt = self.lac4[1].data['Source_Name'] == self.matched['Source_Name']
        if np.sum(srcnamefilt) == 0:
            srcnamefilt = self.lac4_low[1].data['Source_Name'] == self.matched['Source_Name']
            if np.sum(srcnamefilt) == 1:
                self.matched_lac4 = self.lac4_low[1].data[srcnamefilt]
            else:
                self.matched_lac4 = None
        else:
            self.matched_lac4 = self.lac4[1].data[srcnamefilt]

        return(self.matched_lac4[0])

    def get_properties(self):
        self.result['name']  = self.matched['Source_Name']
        if '3FHL' in self.result['name']:
            self.result['redshift'] = self.matched['REDSHIFT']
            if self.matched['NuPeak_obs']<1e14:
                self.result['class'] = 'LSP'
            elif self.matched['NuPeak_obs']<1e15:
                self.result['class'] = 'ISP'
            elif self.matched['NuPeak_obs']<1e17:
                self.result['class'] = 'HSP'
            elif self.matched['NuPeak_obs']>1e18:
                self.result['class'] = 'EHSP'
            else:
                self.result['class'] = ''
        elif '4FGL' in self.result['name']: #
            self.result['redshift'] = self.matched_lac4['Redshift'][0]
            if self.matched_lac4['nu_syn']<1e14:
                self.result['class'] = 'LSP'
            elif self.matched_lac4['nu_syn']<1e15:
                self.result['class'] = 'ISP'
            elif self.matched_lac4['nu_syn']<1e17:
                self.result['class'] = 'HSP'
            elif self.matched_lac4['nu_syn']>1e18:
                self.result['class'] = 'EHSP'
            else:
                self.result['class'] = ''


# Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-

from __future__ import absolute_import, division, unicode_literals, print_function

import os
import re
import itertools
import multiprocessing
import time
import traceback
import six

from ..console import log
from ..publishing import OutputPublisher
from ..step_detect import detect_regressions

from .. import util

class Regressions(OutputPublisher):
    name = "regressions"
    button_label = "Show regressions"
    description = "Display information about recent regressions"

    @classmethod
    def publish(cls, conf, repo, benchmarks, graphs, hash_to_date):
        # Analyze the data in the graphs --- it's been cleaned up and
        # it's easier to work with than the results directly

        regressions = []
        seen = {}
        date_to_hash = dict((d, h) for h, d in six.iteritems(hash_to_date))

        data_filter = _GraphDataFilter(conf, repo, hash_to_date)

        all_params = {}
        for graph in six.itervalues(graphs):
            for key, value in six.iteritems(graph.params):
                all_params.setdefault(key, set())
                if value:
                    all_params[key].add(value)

        last_percentage_time = time.time()

        n_processes = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(n_processes)
        try:
            results = []
            for j, item in enumerate(six.iteritems(graphs)):
                file_name, graph = item
                if 'summary' in graph.params:
                    continue

                # Print progress status
                log.add('.')
                if time.time() - last_percentage_time > 5:
                    log.add('{0:.0f}%'.format(100*j/len(graphs)))
                    last_percentage_time = time.time()

                benchmark_name = os.path.basename(file_name)
                benchmark = benchmarks.get(benchmark_name)
                if not benchmark:
                    continue

                graph_data = data_filter.get_graph_data(graph, benchmark)
                for data in graph_data:
                    results.append((pool.apply_async(_analyze_data, (data,), {}), graph))

                while len(results) > n_processes:
                    r, graph = results.pop(0)
                    cls._insert_regression(regressions, seen, date_to_hash, repo, all_params,
                                           r.get(), graph)

            while results:
                r, graph = results.pop(0)
                cls._insert_regression(regressions, seen, date_to_hash, repo, all_params,
                                       r.get(), graph)
        finally:
            pool.terminate()

        cls._save(conf, {'regressions': regressions})

    @classmethod
    def _insert_regression(cls, regressions, seen, date_to_hash, repo, all_params,
                           result_item, graph):
        j, entry_name, result = result_item
        if result is None:
            return

        # Check which ranges are a single commit
        jumps = result[0]
        for k, jump in enumerate(jumps):
            commit_a = date_to_hash[jump[0]]
            commit_b = date_to_hash[jump[1]]
            spec = repo.get_range_spec(commit_a, commit_b)
            commits = repo.get_hashes_from_range(spec)
            if len(commits) == 1:
                jumps[k] = (None, jump[1])

        # Select unique graph params
        graph_params = {}
        for name, value in six.iteritems(graph.params):
            if len(all_params[name]) > 1:
                graph_params[name] = value

        graph_path = graph.path + '.json'

        # Produce output -- report only one result for each
        # benchmark for each branch
        regression = [entry_name, graph_path, graph_params, j, result]
        key = (entry_name, graph_params.get('branch'))
        if key not in seen:
            regressions.append(regression)
            seen[key] = regression
        else:
            # Pick the worse regression
            old_regression = seen[key]
            prev_result = old_regression[-1]
            if abs(prev_result[1]*result[2]) < abs(result[1]*prev_result[2]):
                old_regression[:] = regression

    @classmethod
    def _save(cls, conf, data):
        fn = os.path.join(conf.html_dir, 'regressions.json')
        util.write_json(fn, data)

def _analyze_data(graph_data):
    """
    Analyze a single time series

    Returns
    -------
    jumps : list of (time_a, time_b)
         List of time pairs, between which there is an upward jump
         in the value.
    cur_value : int
         Most recent value
    best_value : int
         Best value
    """
    try:
        j, entry_name, times, values = graph_data

        v, jump_pos, best_v = detect_regressions(values)
        if v is None:
            return j, entry_name, None

        jumps = []
        for jump_r in jump_pos:
            for r in range(jump_r + 1, len(values)):
                if values[r] is not None:
                    next_r = r
                    break
            else:
                next_r = jump_r + 1
            jumps.append((times[jump_r], times[next_r]))

        return j, entry_name, (jumps, v, best_v)
    except BaseException as exc:
        raise util.ParallelFailure(str(exc), exc.__class__, traceback.format_exc())

class _GraphDataFilter(object):
    """
    Obtain data sets from graphs, following configuration settings.
    """

    def __init__(self, conf, repo, hash_to_date):
        self.conf = conf
        self.repo = repo
        self.hash_to_date = hash_to_date
        self.time_sets = {}

    def get_graph_data(self, graph, benchmark):
        """
        Iterator over graph data sets

        Yields
        ------
        param_idx
            Flat index to parameter permutations for parameterized benchmarks.
            None if benchmark is not parameterized.
        entry_name
            Name for the data set. If benchmark is non-parameterized, this is the
            benchmark name.
        times
            List of times (ints)
        values
            List of benchmark values (floats or Nones)

        """
        series = graph.get_data()

        if benchmark.get('params'):
            param_iter = enumerate(itertools.product(*benchmark['params']))
        else:
            param_iter = [(None, None)]

        for j, param in param_iter:
            if param is None:
                entry_name = benchmark['name']
            else:
                entry_name = benchmark['name'] + '({0})'.format(', '.join(param))

            time_set = self._get_allowed_times(graph, benchmark, entry_name)

            times = [item[0] for item in series if item[0] in time_set]
            if param is None:
                values = [item[1] for item in series if item[0] in time_set]
            else:
                values = [item[1][j] for item in series if item[0] in time_set]

            yield j, entry_name, times, values

    def _get_allowed_times(self, graph, benchmark, entry_name):
        """
        Compute the set of times allowed by asv.conf.json.

        The decision which commits to include is based on commit
        order, not on commit authoring date
        """
        time_set = set(self.hash_to_date.values())

        if graph.params.get('branch'):
            branch_suffix = '@' + graph.params.get('branch')
        else:
            branch_suffix = ''

        for regex, start_commit in six.iteritems(self.conf.regressions_first_commits):
            if re.match(regex, entry_name + branch_suffix):
                if start_commit is None:
                    # Disable regression detection completely
                    return set()

                if self.conf.branches == [None]:
                    key = (start_commit, None)
                else:
                    key = (start_commit, graph.params.get('branch'))

                if key not in self.time_sets:
                    times = set()
                    spec = self.repo.get_new_range_spec(*key)
                    start_hash = self.repo.get_hash_from_name(start_commit)
                    for commit in [start_hash] + self.repo.get_hashes_from_range(spec):
                        time = self.hash_to_date.get(commit[:self.conf.hash_length])
                        if time is not None:
                            times.add(time)
                    self.time_sets[key] = times

                time_set = time_set.intersection(self.time_sets[key])

        return time_set


par_number = 1
Result_dir = '/zhome/13/e/97883/Documents/Speciale_code/Results/Hyperparameter_tuning2/'
data_dir = '/zhome/13/e/97883/Documents/Speciale_code/Data/Classification4_50000.mat'

##
#CHANGES FROM HYPERPARAMETER_TUNING1:
# - Bigger data set (approx 35000 observations)
# - Stratified sampling
# - Balanced classes
# - Possibly bigger networks
# - Batch-sizes restricted to be between 32 and 512
# - L2 regularization on all layers are implemented
# - Saves the best model

#GOALS WITH THIS HYPERPARAMETER TUNING:
# - Reduce avoidable bias from approximately 0.7% to 0% (allowing for bigger models,
#longer training)
#- Reduce variance error from approximately 0.8% to 0% (more data, L2 regularization)

#FUTURE HYPERPARAMETER TUNINGS:
# - Implementing early stopping (?)

#Result_direct = '/zhome/13/e/97883/Documents/Speciale_code/Results/Hyperparameter_tuning2/'

'''
Loading in the data
'''

import scipy.io as sio
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras import optimizers, regularizers
from sklearn.model_selection import train_test_split

#First with limited data (Classification)
#mat_content = sio.loadmat('/Users/Eigil/Dropbox/DTU/Speciale/Data Generation/Classification4_50000.mat')
mat_content = sio.loadmat(data_dir)

X = mat_content['X']
y = mat_content['y'].ravel()

#Computing number of observations for smallest class
counts = np.unique(y, return_counts=True)
n = np.min(counts[1])

mask = np.hstack([np.random.choice(np.where(y == l)[0], n, replace=False)
                      for l in np.unique(y)])

print('Number of discarded observations: '+str(len(y) - len(mask)))
X = X[mask]
y = y[mask]
y = np.reshape(y, (-1, 1))
##Training/Testing set (stratify)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.14, stratify = y, random_state=0)
X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, stratify = y_test, random_state=0)

#Normalising the data
X_train_mean = np.mean(X_train, axis=0)
X_train = X_train - X_train_mean
X_test = X_test - X_train_mean #Differences up to 1e-2
X_val = X_val - X_train_mean

no_samples = 100
batch_size_array = [32, 64, 128, 256, 512]
n1_range = [4, 100]
n2_range = [4, 100]
n3_range = [2, 50]
n4_range = [2, 50]

alpha_range = [-4, -1]
decay_range = [-8, -4] #learning rate decay
l2_range = [-5, -1] #Regularization parameter

number_of_layers = [2, 3, 4]
number_of_epochs = 2000

#Sequential implementation
accuracies = np.empty((no_samples, 1))
scores = np.empty((no_samples, 1))
accuracies[:] = np.NaN
scores[:] = np.NaN
hyperparameters = np.empty((no_samples, 9))
#Hyperparameters: Batch size, alpha, decay, number of layers, n1, n2, n3, n4
hyperparameters[:] = np.NaN

best_acc = 0
best_score = np.inf
best_acc_i = 0
best_score_i = 0
#Can be parallelized
for i in range(no_samples):

    #Draw learning rate
    r = alpha_range[0] + np.random.rand()*(alpha_range[1] - alpha_range[0])
    alpha = 10 ** r

    #Draw number of layers
    index = np.random.randint(0, len(number_of_layers))
    nl = number_of_layers[index]

    #Draw number of neurons
    n1 = np.random.randint(n1_range[0], n1_range[1])
    n2 = np.random.randint(n2_range[0], n2_range[1])
    n3 = np.NaN
    n4 = np.NaN
    if nl > 2:
        n3 = np.random.randint(n3_range[0], n3_range[1])
    if nl > 3:
        n4 = np.random.randint(n4_range[0], n4_range[1])

    #Draw batch size
    index = np.random.randint(0, len(batch_size_array))
    batch_size = batch_size_array[index]

    #Learning rate decay
    r = decay_range[0] + np.random.rand()*(decay_range[1] - decay_range[0])
    decay = 10**r

    #Regularization
    l2 = l2_range[0] + np.random.rand()*(l2_range[1] - l2_range[0])
    l2 = 10**l2
    #Each bin 0.1, 0.01, 0.001 have the same probability. 0 should have the same
    if np.random.binomial(1, p=1/(l2_range[1] - l2_range[0] + 2)) == 1:
        l2 = 0

    #Build model
    model = Sequential()
    model.add(Dense(n1, input_dim=8, activation='relu', kernel_regularizer=regularizers.l2(l2)))
    model.add(Dense(n2, activation='relu'))
    if nl > 2:
        model.add(Dense(n3, activation='relu', kernel_regularizer=regularizers.l2(l2)))
    if nl > 3:
        model.add(Dense(n4, activation = 'relu', kernel_regularizer=regularizers.l2(l2)))
    model.add(Dense(1, activation='sigmoid'))

    adam = optimizers.Adam(learning_rate = alpha, decay=decay)
    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])

    history = model.fit(X_train, y_train, epochs=number_of_epochs, batch_size=batch_size, verbose=0)
    score, acc = model.evaluate(X_val, y_val, verbose=0)

    #Save best model wrt. score
    if score < best_score:
        model.save(filepath='best_score'+str(par_number)+'.h5')
        best_score = score
        best_score_i = i

    if acc > best_acc:
        model.save(filepath='best_acc'+str(par_number)+'.h5')
        best_acc = acc
        best_acc_i = i

    #Sequential implementation
    scores[i, 0] = score
    accuracies[i, 0] = acc
    hyperparameters[i, :] = np.array([batch_size, alpha, decay, l2, nl, n1, n2, n3, n4])
    print('Trying hyperparameter configuration number: '+str(i+1))

#np.savez('RandomSearch.npz', HypPar=hyperparameters, scores=scores, accs = accuracies)
print('Best score index: '+str(best_score_i))
print('Best accuracy index: '+str(best_acc_i))

CSV = np.concatenate((hyperparameters, scores, accuracies), axis=1)
#np.savetxt('test.csv', CSV, delimiter=",", header="batch_size,alpha,decay,l2, nl,n1,n2,n3,n4,score,accuracies")
np.savetxt(Result_dir+'RandomSearch'+str(par_number)+'.csv', CSV, delimiter=",", header="batch_size,alpha,decay,l2,nl,n1,n2,n3,n4,score,accuracies")

#A = np.load('/Users/Eigil/Desktop/RandomSearch.npz')

#'''
#Looking at hyperparameters
#'''
#
#A = []
#for i in range(1, 4+1):
#    A.append(np.loadtxt('/Users/Eigil/Dropbox/DTU/Speciale/Analysis/Hyperparameter_tuning1/RandomSearch'+str(i)+'.csv', delimiter=","))
#
#A = np.vstack(A)
#
#fig, ax = plt.subplots()
#plt.scatter(A[:, 0], A[:, -1], alpha=0.5)
#plt.ylim(0.95, 1)
#ax.set_xscale('log')
#plt.title('Batch size')
#
#
#fig, ax = plt.subplots()
#plt.scatter(A[:, 1], A[:, -1], alpha=0.5)
#plt.ylim(0.95, 1)
#plt.title('Alpha')
#
#fig, ax = plt.subplots()
#plt.scatter(A[:, 2], A[:, -1], alpha=0.5)
#plt.xlim([10**(-8), 10**(-4)])
#plt.ylim([0.95, 1])
#ax.set_xscale('log')
#
#
#fig, ax = plt.subplots()
#plt.scatter(A[:, 3], A[:, -1], alpha=0.5)
#plt.ylim([0.95, 1])
#plt.title('Number of layers')
#
##Finding the 10 highest-yielding hyper-parameters
#n = 10
#indices_acc = (-A[:, -1]).argsort()[0:n]
#indices_score = (A[:, -2]).argsort()[0:n]
#
#A[indices_acc, :]
#
#A[indices_acc[0], :]
#
##Retraining network
#opt_hyperparameters = A[indices_acc[0], :]
#n1_opt = (opt_hyperparameters[4]).astype(int)
#n2_opt = (opt_hyperparameters[5]).astype(int)
#batch_size_opt = (opt_hyperparameters[0]).astype(int)
#alpha_opt = (opt_hyperparameters[1])
#decay_opt = (opt_hyperparameters[2])
#
#model = Sequential()
#model.add(Dense(n1_opt, input_dim=8, activation='relu', kernel_initializer = 'he_normal'))
#model.add(Dense(n2_opt, activation='relu', kernel_initializer = 'he_normal'))
#model.add(Dense(1, activation='sigmoid'))
#adam = optimizers.Adam(learning_rate = alpha_opt, decay=decay_opt)
#model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
#history = model.fit(X_train, y_train, epochs=10000, batch_size=batch_size_opt, verbose=1)
#model = keras.models.load_model('/Users/Eigil/Dropbox/DTU/Speciale/Analysis/Hyperparameter_tuning1/Full_model.hdf5')
#model.evaluate(x=X_test, y=y_test)


import pyaudio
import wave
from timeit import default_timer as timer

# set desired values
start = 7
length = 3

# open wave file
wave_file = wave.open('track.wav', 'rb')

# initialize audio
py_audio = pyaudio.PyAudio()
stream = py_audio.open(format=py_audio.get_format_from_width(wave_file.getsampwidth()),
                       channels=wave_file.getnchannels(),
                       rate=wave_file.getframerate(),
                       output=True)

# skip unwanted frames
n_frames = int(start * wave_file.getframerate())
wave_file.setpos(n_frames)

# write desired frames to audio buffer
n_frames = int(length * wave_file.getframerate())
frames = wave_file.readframes(n_frames)
start = timer()
stream.write(frames)
end = timer()
print(end - start)

# close and terminate everything properly
stream.close()
py_audio.terminate()
wave_file.close()


import xlrd as xl
import pydotplus
import numpy as np

from sklearn import tree

workbook_mat = xl.open_workbook('student/student-mat.xls')
sheet_mat = workbook_mat.sheet_by_index(0)

workbook_por = xl.open_workbook('student/student-por.xls')
sheet_por = workbook_por.sheet_by_index(0)

col14_mat = sheet_mat.col(14)

#Make one big array of all values from both por and mat. We choose to add first the por list and afterwards the mat list
X = []
Failures = []
for i in range(0, sheet_por.row_len(0)):
    if (i != 14):
        X.append(sheet_por.col_values(i, 1))
    else:
        Failures.append(sheet_por.col_values(i, 1))

for i in range(0, sheet_mat.row_len(0)):
    if (i < 14):
        for j in range(1, len(sheet_mat.col_values(i, 1))):
            X[i].append(sheet_mat.cell_value(j, i))
    if (i == 14):
        for j in range(1, len(sheet_mat.col_values(i, 1))):
            Failures[0].append(sheet_mat.cell_value(j, i))
    if (i > 14):
        for j in range(1, len(sheet_mat.col_values(i, 1))):
            X[i-1].append(sheet_mat.cell_value(j, i))

Failures = Failures[0]

#yes/no columns [15, 16, 17, 18, 19, 20, 21, 22, 23]

#Transition from strings to ints
for i in range (0, len(X)):
    a = np.array(X[i])
    if (i == 0):
        b = np.where(a=="GP", 1, 0)
        X[i] = b.tolist()
    elif (i == 1):
        b = np.where(a=="F", 1, 0)
        X[i] = b.tolist()
    elif (i == 3):
        b = np.where(a=="U", 1, 0)
        X[i] = b.tolist()
    elif (i == 4):
        b = np.where(a=="LE3", 1, 0)
        X[i] = b.tolist()
    elif (i == 5):
        b = np.where(a=="T", 1, 0)
        X[i] = b.tolist()
    elif (i == 8 or i == 9):
        tmp = []
        for j in range(0, len(X[i])):
            if (X[i][j] == "teacher"):
                X[i][j] = 0
            elif (X[i][j] == "health"):
                X[i][j] = 1
            elif (X[i][j] == "services"):
                X[i][j] = 2
            elif (X[i][j] == "at_home"):
                X[i][j] = 3
            elif (X[i][j] == "other"):
                X[i][j] = 4
    elif (i == 10):
        tmp = []
        for j in range(0, len(X[i])):
            if (X[i][j] == "home"):
                X[i][j] = 0
            elif (X[i][j] == "reputation"):
                X[i][j] = 1
            elif (X[i][j] == "course"):
                X[i][j] = 2
            elif (X[i][j] == "other"):
                X[i][j] = 3
    elif (i == 11):
        for j in range(0, len(X[i])):
            if (X[i][j] == "mother"):
                X[i][j] = 0
            elif (X[i][j] == "father"):
                X[i][j] = 1
            elif (X[i][j] == "other"):
                X[i][j] = 2
    elif (i == 14 or i == 15 or i == 16 or i == 17 or i == 18 or i == 19 or i == 20 or i == 21):
        b = np.where(a=="yes", 1, 0)
        X[i] = b.tolist()

X2=[]
for i in range(0,len(X[0])):
    tmp=[]
    for j in range(0,31):
        tmp.append(X[j][i])
    X2.append(tmp)

clf = tree.DecisionTreeClassifier(min_samples_split=100)
clf = clf.fit(X2, Failures)

dot_data = tree.export_graphviz(clf, out_file="tree.dot")
# write_pdf didn't work. Therefore we have written the tree to a .dot file and copied the contents
# on graphvizweb.com to see the actual tree.

# graph = pydotplus.graph_from_dot_data(dot_data)
# graph.write_pdf('Images/porDTGraph.pdf')


from API.models import VehicleDetails
from API.serializers import VehicleDetailsSerializer
from rest_framework import generics
from rest_framework.response import Response
from rest_framework import status
from rest_framework.decorators import api_view
from drf_yasg.utils import swagger_auto_schema
from drf_yasg import openapi
from rest_framework import mixins
from rest_framework.parsers import JSONParser
from django.http import JsonResponse

# ====== Vehicle Details Views Starts ========
@api_view(['GET'])
def GetAllVehicleDetailsView(request):
    AllVehicle = VehicleDetails.objects.all()
    serializer_class = VehicleDetailsSerializer(AllVehicle, many=True)
    return Response({"data":serializer_class.data}, status=status.HTTP_200_OK)

@api_view(['GET'])
def GetVehicleByIDView(request):
    # Getting VehicleID in uuid (Queru Param ?vehicle_id=)
    VehicleID = request.query_params.get('vehicle_id')
    # If VehicleID is not provided
    if VehicleID is None:
        return Response({"data": "Vehicle Id Not Provided"}, status=status.HTTP_400_BAD_REQUEST)
    try:
        VehicleData = VehicleDetails.objects.get(vehicleDetailsID=VehicleID)
        serializer = VehicleDetailsSerializer(VehicleData, many=False)
        return Response({"data":serializer.data}, status=status.HTTP_200_OK)
    except Exception as err:
        print(err)
        return Response(str(err), status=status.HTTP_500_INTERNAL_SERVER_ERROR)

@api_view(['POST'])
def CreateVechileDetailsView(request):
    ReqData = request.data
    try:
        # ReqData['password'] = make_password(request.data['password'])
        serializers = VehicleDetailsSerializer(data=ReqData)
        if serializers.is_valid():
            serializers.save()
            return Response(serializers.data,status=status.HTTP_201_CREATED)
        return Response(serializers.errors,status=status.HTTP_400_BAD_REQUEST)
    except Exception as err:
        print(err)
        return Response(serializers.errors, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

@api_view(['DELETE'])
def DeleteVehicleDetailsView(request):
    # Getting Vehicle in uuid (Queru Param ?VehicleId=)
    VehicleID = request.query_params.get('vehicle_id')
    # If UserId is not provided
    if VehicleID is None:
        return Response({"data": "Vehicle Id Not Provided"}, status=status.HTTP_400_BAD_REQUEST)
    try:
        vehmodel = VehicleDetails.objects.get(vehicleDetailsID=VehicleID)
        vehmodel.delete()
        return Response({"data": "VehicleDetails Model Deleted"}, status=status.HTTP_200_OK)
    except Exception as err:
        return Response({"msg": err.message}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)


from base.helpers import model_to_dict
from django.shortcuts import redirect, render
from .form import ArticleForm, ArticleSearchForm
from .model import Article
from base.helpers import HanlderFile, paginator, reverse_url
from base.views import BaseDeleteView, BaseListView, BaseView
from django.contrib import messages

# =========================================================================================================
class ArticleView(BaseView):
    MODULE_NAME = 'Bài viết'
    URL_NAME = 'article'
    TEMPLATE_PATH = 'article/'
    MEDIA_PATH = 'article'

    def init(self, request):
        self.MODEL = Article
        self.params['url_name'] = self.URL_NAME
        return super().init(request)

class ArticleFormView(ArticleView):
    def init(self, request, id=None):
        self.set_template('form.html')
        title = 'Cập nhật' if id else 'Thêm mới'
        self.config_page_header(title, button_params=[
            {'text': 'Quay lại', 'href': reverse_url('article')},
            {'type': 'submit', 'text': 'Lưu'}
        ])
        return super().init(request)

    def get(self, request, id=None):
        item = self.MODEL.objects.get(id=id) if id else None
        self.params['form'] = ArticleForm(initial=model_to_dict(item) if item else {})
        return render(request, self.template, self.params)

    def post(self, request, id=None):
        task = 'change' if id else 'add'
        post_data = self.params.get('post_data')
        image_file = request.FILES.get('upload_image')
        validate_image = None
        if image_file:
            validate_image = HanlderFile.validate_file_input(image_file)
            post_data['image'] = validate_image
        form = ArticleForm(post_data)
        if form.is_valid():
            instance = self.MODEL.objects.get(id=id) if id else None
            if validate_image == True and instance and instance.image: HanlderFile.delete_image(instance.image)
            if image_file: post_data['image'] = HanlderFile.upload_image(image_file, self.MEDIA_PATH).get('dir')
            if self.MODEL.save_item(post_data, {'task': task}):
                messages.add_message(request, messages.SUCCESS, 'Thêm bản ghi thành công.' if task == 'add' else 'Cập nhật bản ghi thành công.')
                return redirect(self.URL_NAME)

        self.params['form'] = form;
        return render(request, self.template, self.params)

# =========================================================================================================
class ArticleListView(BaseListView, ArticleView):
    def init(self, request):
        self.set_template('list.html')
        self.config_page_header('Danh sách', button_params=[
            {'type': 'danger', 'text': 'Xoá', 'add_class': 'action-multi-items action-delete', 'attrs': {'data-bs-toggle': 'modal', 'data-bs-target': '#confirmDeleteModel'}},
            {'type': 'primary', 'text': 'Thêm', 'href': reverse_url('article.add')}
        ])
        return super().init(request)

    def get(self, request, page=1):
        get_data = self.params.get('get_data')
        title = {
            'video': 'Danh sách video',
            'page': 'Danh sách trang',
            'post': 'Danh sách bài đăng',
            None: 'Danh sách'
        }
        self.config_page_header(title[get_data.get('type')], button_params=[
            {'type': 'danger', 'text': 'Xoá', 'add_class': 'action-multi-items action-delete', 'attrs': {'data-bs-toggle': 'modal', 'data-bs-target': '#confirmDeleteModel'}},
            {'type': 'primary', 'text': 'Thêm', 'href': reverse_url('article.add')}
        ])
        self.params['form'] = ArticleSearchForm(get_data)
        self.params = {**self.params, **paginator(self.MODEL.list_item(get_data, {'task': 'list-item'}), {'page': page, **self.params})}
        return render(request, self.template, self.params)

    def post(self, request):
        return render(request, self.template, self.params)

# =========================================================================================================
class ArticleDeleteView(BaseDeleteView, ArticleView): pass


from trafilatura import bare_extraction
from trafilatura.meta import reset_caches

from text_extract.html.abc_html_extract import AbstractHtmlExtractSvc

class TrafilaturaSvc(AbstractHtmlExtractSvc):
    def __init__(self):
        super().__init__()

    def extract_from_html(self, html_str: str):
        extract = bare_extraction(html_str, favor_precision=True)
        reset_caches()
        try:
            return extract['text'].split("\n")
        except:
            return []


#!/usr/bin/env python3
import os
import sys
import json
try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen
import struct
from resources.log import getLogger
from resources.readsettings import ReadSettings
from resources.metadata import MediaType
from resources.mediaprocessor import MediaProcessor

log = getLogger("SickbeardPostProcess")

log.info("Sickbeard extra script post processing started.")

try:
    settings = ReadSettings()

    if len(sys.argv) > 4:
        inputfile = sys.argv[1]
        original = sys.argv[2]
        tvdb_id = int(sys.argv[3])
        season = int(sys.argv[4])
        episode = int(sys.argv[5])

        log.debug("Input file: %s." % inputfile)
        log.debug("Original name: %s." % original)
        log.debug("TVDB ID: %s." % tvdb_id)
        log.debug("Season: %s episode: %s." % (season, episode))

        mp = MediaProcessor(settings)

        success = mp.fullprocess(inputfile, MediaType.TV, tvdbid=tvdb_id, season=season, episode=episode, original=original)
        if success:
            try:
                protocol = "https://" if settings.Sickbeard['ssl'] else "http://"
                host = settings.Sickbeard['host']  # Server Address
                port = settings.Sickbeard['port']  # Server Port
                apikey = settings.Sickbeard['apikey']  # Sickbeard API key
                webroot = settings.Sickbeard['webroot']  # Sickbeard webroot

                sickbeard_url = protocol + host + ":" + str(port) + webroot + "/api/" + apikey + "/?cmd=show.refresh&tvdbid=" + str(tvdb_id)

                refresh = json.load(urlopen(sickbeard_url))
                for item in refresh:
                    log.debug(refresh[item])
            except (IOError, ValueError):
                try:
                    log.debug("Sickbeard refresh failed, trying to fall back to Sickrage settings if available")
                    protocol = "https://" if settings.Sickrage['ssl'] else "http://"
                    host = settings.Sickrage['host']  # Server Address
                    port = settings.Sickrage['port']  # Server Port
                    apikey = settings.Sickrage['apikey']  # Sickbeard API key
                    webroot = settings.Sickrage['webroot']  # Sickbeard webroot

                    sickbeard_url = protocol + host + ":" + str(port) + webroot + "/api/" + apikey + "/?cmd=show.refresh&tvdbid=" + str(tvdb_id)

                    refresh = json.load(urlopen(sickbeard_url))
                    for item in refresh:
                        log.debug(refresh[item])
                except (IOError, ValueError):
                    log.exception("Couldn't refresh Sickbeard/Sickrage, check your autoProcess.ini settings.")
    else:
        log.error("Not enough command line arguments present %s." % len(sys.argv))
        sys.exit(1)
except:
    log.exception("Unexpected exception.")
    sys.exit(1)


"""a trivial demo, with one Linear layer"""
from torch import tensor
from torchnetjson.builder import build_net
module_dict = {
    'fc1': {'name':'torch.nn.linear',
           'params': {'in_features': 3, 'out_features': 5},
           'init': None},
    'fc2': {'name':'torch.nn.linear',
           'params': {'in_features': 5, 'out_features': 3},
           'init': None}
}

op_spec_list = [
    {'name': 'module',
     'args': ['fc1'],
     'kwargs': {},
     },
    {'name': 'module',
     'args': ['fc2'],
     'kwargs': {},
     },
]

op_list = [
    {
        'name': 'sequential',
        'args': [op_spec_list,],
        'kwargs': {},
        'in': 'input0',
        'out': 'out',
    },
    {
        'name': 'detach',
        'args': [],
        'kwargs': {},
        'in': 'out',
        'out': 'out_detached',
    }
]

param_dict = {
    'module_dict': module_dict,
    'op_list': op_list,
    'out': 'out_detached'
}

net = build_net(param_dict)

input_this = tensor([1.0, 2.0, 3.0])
output_this = net(input_this, verbose=True).detach()

print(output_this, type(output_this))


from django.urls import include, path, re_path

from . import views

urlpatterns = [
    path('wiki/<str:title>', views.content),
    path('wiki/<str:title>/edit-entry', views.edit_entry, name='edit_entry'),
    path('wiki/search/', views.get_search),
    path('wiki/random/', views.random_page, name='random'),
    path('wiki/create-new-page/', views.create_new_page, name='create_new_page'),
    path('', views.index, name='index')
]


import os
import sys
import time
from typing import List

from bs4 import BeautifulSoup
from progressbar import ProgressBar

from RatS.base.base_site import BaseSite
from RatS.base.movie_entity import Movie, SiteSpecificMovieData, Site

class RatingsParser:
    def __init__(self, site: BaseSite, args):
        self.site: BaseSite = site
        self.args = args
        if not self.site.CREDENTIALS_VALID:
            return

        self.movies: List[Movie] = []
        self.movies_count: int = 0

        self.site.open_url_with_521_retry(self.site.MY_RATINGS_URL)

        self.exports_folder: str = os.path.abspath(
            os.path.join(
                os.path.dirname(__file__), os.pardir, os.pardir, "RatS", "exports"
            )
        )
        if not os.path.exists(self.exports_folder):
            os.makedirs(self.exports_folder)

        self.progress_bar = None

    def parse(self):
        iteration = 0
        while True:
            iteration += 1
            try:
                self._parse_ratings()
                break
            except AttributeError as e:
                if iteration > 10:
                    raise e
                time.sleep(iteration * 2)
                continue

        self.site.browser_handler.kill()
        return self.movies

    def _parse_ratings(self):
        time.sleep(1)
        movie_ratings_page = BeautifulSoup(self.site.browser.page_source, "html.parser")

        pages_count: int = self._retrieve_pages_count_and_movies_count(
            movie_ratings_page
        )
        if self.args and self.args.verbose and self.args.verbose >= 3:
            sys.stdout.write(
                "\r\n ================================================== \r\n"
            )
            sys.stdout.write(self.site.browser.current_url)
            sys.stdout.write(
                f"\r\n ===== {self.site.site_displayname}: getting page count: {pages_count} \r\n"
            )
            sys.stdout.write(
                f"\r\n ===== {self.site.site_displayname}: getting movies count: {self.movies_count} \r\n"
            )
            sys.stdout.write(
                "\r\n ================================================== \r\n"
            )
            sys.stdout.flush()

        sys.stdout.write(
            f"\r===== {self.site.site_displayname}: Parsing {pages_count} pages"
            f" with {self.movies_count} movies in total\r\n"
        )
        sys.stdout.flush()

        for page_number in range(1, pages_count + 1):
            self.site.open_url_with_521_retry(self._get_ratings_page(page_number))
            movie_listing_page = BeautifulSoup(
                self.site.browser.page_source, "html.parser"
            )
            self._parse_movie_listing_page(movie_listing_page)

    def _retrieve_pages_count_and_movies_count(self, movie_ratings_page) -> int:
        pages_count = self._get_pages_count(movie_ratings_page)
        self.movies_count = self._get_movies_count(movie_ratings_page)
        return pages_count

    @staticmethod
    def _get_pages_count(movie_ratings_page) -> int:
        raise NotImplementedError("This is not the implementation you are looking for.")

    @staticmethod
    def _get_movies_count(movie_ratings_page) -> int:
        raise NotImplementedError("This is not the implementation you are looking for.")

    def _get_ratings_page(self, page_number: int):
        raise NotImplementedError("This is not the implementation you are looking for.")

    def _parse_movie_listing_page(self, movie_listing_page):
        movies_tiles = self._get_movie_tiles(movie_listing_page)
        for movie_tile in movies_tiles:
            movie = self._parse_movie_tile(movie_tile)
            if movie:
                self.movies.append(movie)
            self.print_progress(movie)

    def print_progress(self, movie: Movie):
        if self.args and self.args.verbose and self.args.verbose >= 2:
            sys.stdout.write(
                f"\r===== {self.site.site_displayname}: [{len(self.movies)}/{self.movies_count}] parsed {movie} \r\n"
            )
            sys.stdout.flush()
        elif self.args and self.args.verbose and self.args.verbose >= 1:
            sys.stdout.write(
                f"\r===== {self.site.site_displayname}: [{len(self.movies)}/{self.movies_count}]"
                f" parsed {movie.title} ({movie.year}) \r\n"
            )
            sys.stdout.flush()
        else:
            self._print_progress_bar()

    def _print_progress_bar(self):
        if not self.progress_bar:
            self.progress_bar = ProgressBar(
                max_value=self.movies_count, redirect_stdout=True
            )
        if len(self.movies) >= self.movies_count:
            self.progress_bar.finish()
        else:
            self.progress_bar.update(len(self.movies))

    @staticmethod
    def _get_movie_tiles(movie_listing_page):
        raise NotImplementedError("This is not the implementation you are looking for.")

    def _parse_movie_tile(self, movie_tile):
        movie = Movie(title=self._get_movie_title(movie_tile))
        movie_id = self._get_movie_id(movie_tile)
        site_specific_movie_data = SiteSpecificMovieData(
            id=movie_id,
            url=self._get_movie_url(movie_tile),
        )
        movie.site_data[self.site.site] = site_specific_movie_data

        self._go_to_movie_details_page(movie)
        time.sleep(1)

        iteration = 0
        while True:
            iteration += 1
            try:
                self.parse_movie_details_page(movie)
                break
            except AttributeError as e:
                if iteration > 10:
                    raise e
                time.sleep(iteration * 1)
                continue

        return movie

    def _go_to_movie_details_page(self, movie: Movie):
        self.site.open_url_with_521_retry(movie.site_data[self.site.site].url)

    @staticmethod
    def _get_movie_title(movie_tile):
        raise NotImplementedError("This is not the implementation you are looking for.")

    @staticmethod
    def _get_movie_id(movie_tile):
        raise NotImplementedError("This is not the implementation you are looking for.")

    @staticmethod
    def _get_movie_url(movie_tile):
        raise NotImplementedError("This is not the implementation you are looking for.")

    def parse_movie_details_page(self, movie: Movie):
        raise NotImplementedError("This is not the implementation you are looking for.")

    def _parse_external_links(self, movie: Movie, movie_details_page):
        external_links = self._get_external_links(movie_details_page)
        for link in external_links:
            movie_link = link["href"].strip("/").replace("http://", "https://")
            if "imdb.com" in link["href"] and "find?" not in link["href"]:
                movie.site_data[Site.IMDB] = SiteSpecificMovieData(
                    id=movie_link.split("/")[-1],
                    url=movie_link,
                )
            elif "themoviedb.org" in link["href"]:
                movie.site_data[Site.TMDB] = SiteSpecificMovieData(
                    id=movie_link.split("/")[-1],
                    url=movie_link,
                )

    @staticmethod
    def _get_external_links(movie_details_page):
        raise NotImplementedError("This is not the implementation you are looking for.")


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from spacesApp.models import Space
from spacesApp.forms import SpaceForm
from reservationsApp.models import Reservation
from django.db import models
from datetime import timedelta
import datetime
from django.utils.timezone import localtime

import random, os, re
import pytz
from django.contrib import messages

@login_required
def space_data(request, space_id, date=None):
	try:
		space = Space.objects.get(id=space_id)
		space_request(request, space_id)

		if date:
			current_date = date
			current_week = datetime.datetime.strptime(current_date,
			                                          "%Y-%m-%d").date().isocalendar()[
				1]
		else:
			try:
				current_week = datetime.datetime.strptime(request.GET["date"],
				                                          "%Y-%m-%d").date().isocalendar()[
					1]
				current_date = request.GET["date"]
			except:
				current_week = datetime.date.today().isocalendar()[1]
				current_date = datetime.date.today().strftime("%Y-%m-%d")

		delta = (datetime.datetime.strptime(current_date,
		                                    "%Y-%m-%d").isocalendar()[2]) - 1
		monday = ((datetime.datetime.strptime(current_date,
		                                      "%Y-%m-%d") - datetime.timedelta(
			days=delta)).strftime("%d/%m/%Y"))
		move_controls = list()
		move_controls.append((datetime.datetime.strptime(current_date,
		                                                 "%Y-%m-%d") + datetime.timedelta(
			weeks=-4)).strftime("%Y-%m-%d"))
		move_controls.append((datetime.datetime.strptime(current_date,
		                                                 "%Y-%m-%d") + datetime.timedelta(
			weeks=-1)).strftime("%Y-%m-%d"))
		move_controls.append((datetime.datetime.strptime(current_date,
		                                                 "%Y-%m-%d") + datetime.timedelta(
			weeks=1)).strftime("%Y-%m-%d"))
		move_controls.append((datetime.datetime.strptime(current_date,
		                                                 "%Y-%m-%d") + datetime.timedelta(
			weeks=4)).strftime("%Y-%m-%d"))

		colores = {'A': 'rgba(0,153,0,0.7)',
		           'P': 'rgba(51,51,204,0.7)'}
		reservations = Reservation.objects.filter(space=space,
			starting_date_time__week=current_week, state__in=['P', 'A'])
		res_list = []
		for i in range(5):
			res_list.append(list())
		for r in reservations:
			try:
				print(r)
				reserv = []
				reserv.append(r.space.name)
				reserv.append(localtime(r.starting_date_time).strftime("%H:%M"))
				reserv.append(localtime(r.ending_date_time).strftime("%H:%M"))
				reserv.append(colores[r.state])
				res_list[r.starting_date_time.isocalendar()[2] - 1].append(reserv)

			except:
				pass

		context = {
			'space': space,
			'actual_monday': monday,
			'controls': move_controls,
			'reservations': res_list,
		}

		return render(request, 'space_data.html', context)
	except Exception as e:
		return redirect('/')

def verificar_horario_habil(init, end, space_id):
	if init.hour < 9 or end.hour > 18:
		return False

	if Reservation.objects.filter(starting_date_time__lte=init,
	                              ending_date_time__gte=init, space=space_id)\
			.exclude(state='R'):
		return False

	if Reservation.objects.filter(starting_date_time__gte=init,
	                                  ending_date_time__lte=end, space=space_id)\
			.exclude(state='R'):
		return False

	if init.weekday() > 4: # fin de semana
		return False

	return True

def verificar_espacio_quincho(init, end, name):
	now = datetime.datetime.now()
	if re.search('quincho', name, re.IGNORECASE):
		return init - now >= timedelta(hours=24)
	return True

def space_request(request, space_id):
	if request.method == 'POST' and 'picked-date' in request.POST:
		space = Space.objects.get(id=space_id)

		if request.user.enabled:
			try:
				if space.state == 'R':
					messages.warning(request, 'Sala en reparación, no se puede reservar')
					return

				string_inicio = request.POST['picked-date'] + " " + \
				                request.POST['picked-start-time']
				start_date_time = datetime.datetime.strptime(string_inicio,
				                                    '%d/%m/%Y %H:%M')
				string_fin = request.POST['picked-date'] + " " + request.POST[
					'picked-end-time']
				end_date_time = datetime.datetime.strptime(string_fin, '%d/%m/%Y %H:%M')

				if start_date_time > end_date_time:
					messages.warning(request,
					                 'La reserva debe terminar después de iniciar.')
				elif (start_date_time - datetime.datetime.now() < timedelta(hours=1)) and (not re.search('quincho', space.name, re.IGNORECASE)):
					messages.warning(request,
					                 'Los pedidos (menos el Quincho) deben ser hechos al menos con una hora de anticipación.')
				elif start_date_time.date() != end_date_time.date():
					messages.warning(request,
					                 'Los pedidos deben ser devueltos el mismo día que se entregan.')
				elif not verificar_horario_habil(start_date_time, end_date_time, space_id):
					messages.warning(request,
					                 'Los pedidos deben ser hechos en horario hábil.')
				elif not verificar_espacio_quincho(start_date_time, end_date_time, space.name):
					messages.warning(request, 'El quincho debe reservarse 24 antes.')
				else:
					reservation = Reservation(space=space,
					                          starting_date_time=start_date_time,
					                          ending_date_time=end_date_time,
					                          user=request.user)
					reservation.save()
					messages.success(request, 'Pedido realizado con éxito')
			except Exception as e:
				print(e)
				messages.warning(request, 'Ingrese una fecha y hora válida.')
		else:
			messages.warning(request,
			                 'Usuario no habilitado para pedir préstamos')

@login_required
def space_data_admin(request, space_id):
	if not request.user.is_staff:
		return redirect('/')
	else:
		try:
			space = Space.objects.get(id=space_id)
			context = {
				'space': space
			}
			return render(request, 'space_data_admin.html', context)
		except:
			return redirect('/')

@login_required
def space_data_admin_create(request):
    if not request.user.is_staff:
        return redirect('/')

    if request.method == "GET":
        context = {"states" : Space.STATES}
        return render(request, 'space_data_admin_create.html', context)
    if request.method == "POST":
        form = SpaceForm(request.POST, request.FILES)
        if form.is_valid():
            form.save()
            return redirect('/admin/items-panel')
        else:
            context = {"states" : Space.STATES, "error": "Datos no validos, porfavor intente otra vez"}
            return redirect('/admin/items-panel')

@login_required
def space_edit_name(request, space_id):
	if request.method == "POST":
		a = Space.objects.get(id=space_id)
		a.name = request.POST["name"]
		a.save()
	return redirect('/space/' + str(space_id) + '/edit')

@login_required
def space_edit_image(request, space_id):
	if request.method == "POST":
		u_file = request.FILES["image"]
		extension = os.path.splitext(u_file.name)[1]
		a = Space.objects.get(id=space_id)
		a.image.save(str(space_id) + "_image" + extension, u_file)
		a.save()

	return redirect('/space/' + str(space_id) + '/edit')

@login_required
def space_edit_description(request, space_id):
	if request.method == "POST":
		a = Space.objects.get(id=space_id)
		a.description = request.POST["description"]
		a.save()

	return redirect('/space/' + str(space_id) + '/edit')

@login_required
def space_edit_state(request, space_id):
	if request.method == "POST" \
			and request.POST["state"] in ['D', 'P', 'R', 'L']:
		a = Space.objects.get(id=space_id)
		a.state = request.POST["state"]
		a.save()

	return redirect('/space/' + str(space_id) + '/edit')

@login_required
def space_edit_capacity(request, space_id):
	if request.method == "POST":
		a = Space.objects.get(id=space_id)
		a.capacity = request.POST["capacidad"]
		a.save()

	return redirect('/space/' + str(space_id) + '/edit')


"""
    Project base URL configuration
    ==============================

    For more information on this file, see https://docs.djangoproject.com/en/1.10/topics/http/urls/

"""

from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls.i18n import i18n_patterns
from django.contrib import admin

urlpatterns = [
    # Internationalization.
    url(r'^i18n/', include('django.conf.urls.i18n')),

    # Apps
    url(r'^oidc/', include('oidc_rp.urls')),
]

urlpatterns += i18n_patterns(
    # Admin
    url(r'^admin/', admin.site.urls),

    # Apps
    url(r'^', include('example.public.urls')),
)

if settings.DEBUG:
    # In DEBUG mode, serve media files through Django.
    from django.contrib.staticfiles.urls import staticfiles_urlpatterns
    from django.views import static
    urlpatterns += staticfiles_urlpatterns()
    # Remove leading and trailing slashes so the regex matches.
    media_url = settings.MEDIA_URL.lstrip('/').rstrip('/')
    urlpatterns += [
        url(r'^%s/(?P<path>.*)$' % media_url, static.serve,
            {'document_root': settings.MEDIA_ROOT}),
    ]


## My solution
from collections import OrderedDict
from operator import itemgetter
from collections import Counter

# def non_repeating(given_string):
#     if len(given_string) > 0:
#         my_dict = OrderedDict()
#
#         for ii in range(len(given_string)):
#             if given_string[ii] in my_dict:
#                 my_dict[given_string[ii]] += 1
#             else:
#                 my_dict[given_string[ii]] = 1
#
#         if 1 in my_dict.values():
#             # sort the dict
#             sorted_dict = sorted(my_dict.items(),key = itemgetter(1))
#             return sorted_dict[0][0]
#         else:
#             return None
#
#     else:
#         return None

## solution given in lesson
# # Implement your function below.
# def non_repeating(given_string):
    # char_count = {}
    # for c in given_string:
        # if c in char_count:
            # char_count[c] += 1
        # else:
            # char_count[c] = 1
    # for c in given_string:
        # if char_count[c] == 1:
            # return c
    # return None

## My second more elegant solution
def non_repeating(s):
    c=Counter(s)
    repeat=[]
    for el in c:
        if c[el] == 1:
            repeat.append(el)
            return el
    return None

print(non_repeating("abcab")) # should return 'c'
print(non_repeating("abab")) # should return None
print(non_repeating("aabbbc")) # should return 'c'
print(non_repeating("aabbdbc")) # should return 'd'


# coding: utf-8
# Tom Barclay tom@tombarlay.com
# # Simulate exoplanet yield from TESS
# The purpose of this code is to simulate the exoplanet yield from the TESS Mission.
# We do this by taking the various fields that TESS observes and, using a galaxy model,
#   put planets orbiting the stars and see whether we can detect that planet.

from __future__ import division, print_function
import numpy as np
import pandas as pd
import astroquery
import matplotlib.pyplot as plt
import glob
import matplotlib
matplotlib.style.use('ggplot')

import occSimFuncs as occFunc
from numpy.random import poisson, beta, uniform

#get_ipython().magic(u'matplotlib inline')

#constants
msun = 1.9891E30
rsun = 695500000.
G = 6.67384E-11
AU = 149597870700.

# lets read our galaxt model files

def add_stellar_props(df):
    df['isMdwarf'] = pd.Series((df.CL == 5) & (df.Typ >= 7.), name='isMdwarf' )
    df['isGiant'] = pd.Series((df.CL < 5), name='isGiant' )
    df['I'] = pd.Series(-1. * (df.VI - df.V), name='I')
    df['Teff'] = pd.Series(10**df.LTef , name='Teff')

    g = 10**df.logg * 0.01
    df['Radius'] = pd.Series(np.sqrt(G*df.Mass*msun / g) / rsun, name='Radius')

    return df

def make_fullfov_df(df,consts):
    multiple = consts['galmodarea'] / consts['simsize']
    numstars = int(df.shape[0] * multiple)
    rows = np.random.choice(df.index.values, size=numstars)
    newq = df.ix[rows]

    return newq.set_index(np.arange(newq.shape[0]))

def make_allplanets_df(df,starid_zp):
    df['planetRadius'] = pd.Series()
    df['planetPeriod'] = pd.Series()
    df['starID'] = pd.Series()

    newdf = pd.DataFrame(columns=df.columns)
    starID = starid_zp
    for thisRow in np.arange(df.shape[0]):
        if df.loc[thisRow,'isMdwarf']:
            radper = occFunc.Dressing15_select(df.loc[thisRow, 'Nplanets'])
            if df.loc[thisRow,'Nplanets'] == 0:
                continue
            elif df.loc[thisRow,'Nplanets'] == 1:
                df.loc[thisRow,'planetRadius'] = radper[0]
                df.loc[thisRow,'planetPeriod'] = radper[1]
                df.loc[thisRow,'starID'] = starID
                newdf = newdf.append(df.loc[thisRow])
                starID +=1
            elif df.loc[thisRow,'Nplanets'] >= 2:
                df.loc[thisRow,'starID'] = starID
                for p in np.arange(df.loc[thisRow,'Nplanets']):
                    df.loc[thisRow,'planetRadius'] = radper[0][p]
                    df.loc[thisRow,'planetPeriod'] = radper[1][p]
                    newdf = newdf.append(df.loc[thisRow])
                starID +=1

        elif not df.loc[thisRow,'isMdwarf']:
            radper = occFunc.Fressin13_select(df.loc[thisRow, 'Nplanets'])
            if df.loc[thisRow,'Nplanets'] == 0:
                continue
            elif df.loc[thisRow,'Nplanets'] == 1:
                df.loc[thisRow,'planetRadius'] = radper[0]
                df.loc[thisRow,'planetPeriod'] = radper[1]
                df.loc[thisRow,'starID'] = starID
                newdf = newdf.append(df.loc[thisRow])
                starID +=1
            elif df.loc[thisRow,'Nplanets'] >= 2:
                df.loc[thisRow,'starID'] = starID
                for p in np.arange(df.loc[thisRow,'Nplanets']):
                    df.loc[thisRow,'planetRadius'] = radper[0][p]
                    df.loc[thisRow,'planetPeriod'] = radper[1][p]
                    newdf = newdf.append(df.loc[thisRow])
                starID += 1

    newdf.set_index(np.arange(newdf.shape[0]), inplace=True)

    return newdf, starID

def make_allplanets_df_vec(df,starid_zp):
    # lets refector the above code to make it array operations
    totalRows = df['Nplanets'].sum()

    df['planetRadius'] = pd.Series()
    df['planetPeriod'] = pd.Series()
    df['starID'] = pd.Series()

    radper_dressing = occFunc.Dressing15_select(totalRows)
    radper_fressin = occFunc.Fressin13_select(totalRows)

    #we need an array of indices
    totalRows = df['Nplanets'].sum()
    rowIdx = np.repeat(np.arange(df.shape[0]),np.array(df.Nplanets.values))

    newdf = df.iloc[rowIdx]
    newdf.starID = rowIdx + starid_zp

    newdf.planetRadius = np.where(newdf.isMdwarf,radper_dressing[0],radper_fressin[0])
    newdf.planetPeriod = np.where(newdf.isMdwarf,radper_dressing[1],radper_fressin[1])
    newdf.set_index(np.arange(newdf.shape[0]), inplace=True)

    return newdf, newdf.starID.iloc[-1]

if __name__ == '__main__':

    # the besancon galaxy file column names
    names = ['Dist','Mv','CL','Typ','LTef','logg','Age',
             'Mass','BV','UB','VI','VK','V','FeH',
             'l','b','Av','Mbol']

    # some constants we need
    consts =  {'sigma_threshold': 10.,
               'simsize': 8, #size of the galmod field in sq deg
                'full_fov': True, # if true do whole 24x24deg ccd
              }

    galmodfiles = glob.glob('../data/besmod*.csv')

    #loop over every field
    starID = 0
    sumq = 0

    for thisGalmodidx in range(len(galmodfiles)):

        consts['galmodarea'] = np.genfromtxt('bess_reas.txt',usecols=2)[thisGalmodidx]
        consts['ra'] = np.genfromtxt('bess_reas.txt',usecols=0)[thisGalmodidx]
        consts['dec'] = np.genfromtxt('bess_reas.txt',usecols=1)[thisGalmodidx]
        consts['tess_ccd'] = np.genfromtxt('bess_reas.txt',usecols=3)[thisGalmodidx]

        intial_q = pd.read_csv(galmodfiles[thisGalmodidx], skiprows=2, names=names)
        intial_q = add_stellar_props(intial_q)

        # we previously got the besancon models, they are in the directory ../data/
        # We also saved the areas for each field in bess_reas.txt

        # We are doing this in a monte carlo fashion. Our outer loop is each field.
        # The row closest to the equator is easiest because there is no overlap.
        # We saved a few functions in occSimFuncs.py

        #make the catalog equal to the full fov area
        if consts['full_fov'] & (consts['simsize'] < consts['galmodarea']):
            q = make_fullfov_df(intial_q,consts)

        elif (simsize > consts['galmodarea']):
            raise('Galmod area is too small!')

        else:
            q = intial_q

        # here is how we calculate the observations length
        # - all field on TESS ccd 1 will have an obselen of 27 days
        # - targets on CCD 2 will have either a 54 days or 27 days
        #        - fraction with 54 days x = ((24**2) - A) / (2*A)
        #        - fraction with 27 days = A* (1-x)
        # - targets of CCD are harder
        # for now let's do the same things as CCD2 but THIS IS NOT RIGHT
        #   because some of the targets have obslen = 81 days
        # - targets on CCD 4 have obslen = 351

        q['ra'] = np.zeros(q.shape[0]) + consts['ra']
        q['dec'] = np.zeros(q.shape[0]) + consts['dec']
        q['tess_ccd'] = np.zeros(q.shape[0]) + consts['tess_ccd']
        if consts['tess_ccd'] == 1:
            consts['obslen'] = 27 # the days we observe field, needs to change
            q['obslen'] = np.zeros(q.shape[0]) + consts['obslen']
        elif (consts['tess_ccd'] == 2) or (consts['tess_ccd'] == 3):
            consts['obslen'] = [27,54]
            x_54 = ((24**2) - consts['galmodarea']) / (2*consts['galmodarea'])
            x_27 = consts['galmodarea'] * (1-x_54)
            idx_54 = np.random.choice(np.arange(q.shape[0]), size=x_54*q.shape[0], replace=False)
            idx_54_bool = np.in1d(np.arange(q.shape[0]),idx_54)
            q['obslen'] = np.where(idx_54_bool,54,27)
        elif consts['tess_ccd'] == 4:
            consts['obslen'] = 351
            q['obslen'] = np.zeros(q.shape[0]) + consts['obslen']

        #some planet parameters we will need later
        q['cosi'] = pd.Series(np.random.random(size=q.shape[0]),name='cosi')
        # this cosi will be the same for every planet in the system

        q['noise_level'] = occFunc.TESS_noise_1h(q.I)

        np_fgk = poisson(lam=0.689,size=q.shape[0])
        np_m = poisson(lam=2.5,size=q.shape[0])
        q['Nplanets'] = pd.Series(np.where(q.isMdwarf, np_m,np_fgk), name='Nplanets')

        # draw a bunch of planest and accociate them with each star
        newDF, starID = make_allplanets_df_vec(q, starID)

        newDF['T0'] = uniform(0,1,size=newDF.shape[0]) * newDF.planetPeriod

        nt1 = np.floor((newDF['obslen']-newDF.T0) / newDF.planetPeriod)

        newDF['Ntransits'] = np.where(newDF['obslen'] <= newDF.planetPeriod, nt1+1,0)
        newDF['ars'] = occFunc.per2ars(newDF.planetPeriod,newDF.Mass,newDF.Radius)
        newDF['ecc'] = pd.Series(beta(1.03,13.6,size=newDF.shape[0]),name='ecc', ) # ecc dist from Van Eylen 2015
        newDF['omega'] = pd.Series(uniform(-np.pi,np.pi,size=newDF.shape[0]),name='omega')
        newDF['rprs'] = occFunc.get_rprs(newDF.planetRadius,newDF.Radius)
        newDF['impact'] = newDF.cosi * newDF.ars * ((1-newDF.ecc**2)/1+newDF.ecc*np.sin(newDF.omega)) # cite Winn
        newDF['duration'] = occFunc.get_duration(newDF.planetPeriod,newDF.ars,cosi=newDF.cosi, b=newDF.impact,
                                        rprs=newDF.rprs) # cite Winn
        newDF['duration_correction'] = np.sqrt(newDF.duration * 24.) # correction for CDPP because transit dur != 1 hour
        newDF['transit_depth']  = occFunc.get_transit_depth(newDF.planetRadius,newDF.Radius)

        # now lets see if those planets are detected

        newDF['needed_for_detection'] = (newDF.transit_depth * newDF.duration_correction *
                    np.sqrt(newDF.Ntransits)) / consts['sigma_threshold']
        newDF['has_transits']  = (newDF.ars > 1.0) & (newDF.impact < 1.0)

        newDF['detected'] = (newDF.noise_level < newDF.needed_for_detection) & (newDF.Ntransits >= 3) & (newDF.planetRadius > 0.0) & newDF.has_transits

        total_planets = newDF.detected.sum()

        try:
            detected_DF = pd.concat([detected_DF,newDF[newDF.detected == True]])
        except:
            detected_DF = newDF[newDF.detected == True]

        sumq += q.shape[0]


#!/usr/bin/python

import rospy
import roslib
import sensor_msgs
#import opencv2
from cv_bridge import CvBridge
import std_msgs
import sys
import cv2
import sensor_msgs.point_cloud2 as pc2
from sensor_msgs.msg import PointCloud2, PointField
from reconstruction.msg import ImageStamped
from std_msgs.msg import Header
import numpy as np
from geometry_msgs.msg import Pose

points = []
K = np.eye(3)

f1 = 1.0
pp1 = (0.0,0.0)
f2 = 1.0
pp2 = (0.0,0.0)
#obviously wrong, need ammar to tell me

K = np.eye(3)

#tunable params
MIN_MATCH_COUNT = 50

#sift params
sift_N_octave_layers = 10
sift_edge_threshold = 100
sift_sigma = 1

#flann params
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees=10)
search_params = dict(checks=50)
K_neighbors = 2
dist_thresh = 0.7

def pose_to_tf(pose):

	quat = pose.orientation

	tf_mat = np.zeros((4,4))

	tf_mat[0,0] = 1 - 2 * quat.y ** 2 - 2 ** quat.z ** 2
	tf_mat[0,1] = 2 * quat.x * quat.y - 2 * quat.z * quat.w
	tf_mat[0,2] = 2 * quat.x * quat.z + 2 * quat.y * quat.w

	tf_mat[1,0] = 2 * quat.x * quat.y + 2 * quat.z * quat.w
	tf_mat[1,1] = 1 - 2 * quat.x ** 2 - 2 * quat.z ** 2
	tf_mat[1,2] = 2 * quat.y * quat.z - 2 * quat.x * quat.w

	tf_mat[2,0] = 2 * quat.x * quat.z - 2 * quat.y * quat.w
	tf_mat[2,1] = 2 * quat.y * quat.z + 2 * quat.x * quat.w
	tf_mat[2,2] = 1 - 2 * quat.x ** 2 - 2 * quat.y ** 2

	tf_mat[0,3] = pose.position.x
	tf_mat[1,3] = pose.position.y
	tf_mat[2,3] = pose.position.z
	tf_mat[3,3] = 1
	return tf_mat

def make3d(img1, tfMat1, img2, tfMat2):

	projMat1 = np.dot(K, np.linalg.inv(tfMat1)[:3,:])
	projMat2 = np.dot(K, np.linalg.inv(tfMat2)[:3,:])

	#projection matrices are P = K * [R|t] where K is a 3x3 of the intrinsic params of each camera
	res_img_sift1 = img1
	res_img_sift2 = img2
	#implementign sift feature extraction
	sift = cv2.xfeatures2d.SIFT_create(nOctaveLayers=sift_N_octave_layers, edgeThreshold=sift_edge_threshold, sigma=sift_sigma) #implemeting sift feature extraction
	kp1,des1 = sift.detectAndCompute(img1,None)
	kp2,des2 = sift.detectAndCompute(img2,None)
	#to visualize while dbg
	# img_sift1 = cv2.drawKeyPoints(img1,kp1,None,flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

	#implementing FLANN matching of sift features
	flann = cv2.FlannBasedMatcher(index_params,search_params)
	matches = flann.knnMatch(des1,des2,k=K_neighbors)

	#storing good matches
	good = []

	for m,n in matches:
		if m.distance < dist_thresh * n.distance:
			good.append(m)

	if len(good) > MIN_MATCH_COUNT:
		src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1,1,2)
		dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1,1,2)

		src_pts_norm = cv2.undistortPoints(src_pts, cameraMatrix=K,distCoeffs=None)
		dst_pts_norm = cv2.undistortPoints(dst_pts, cameraMatrix=K,distCoeffs=None)

		E, mask = cv2.findEssentialMat(src_pts_norm,dst_pts_norm,focal=f1, pp=pp1,method=cv2.RANSAC,prob=0.999,threshold=3.0)
		points,R,t,mask = cv2.recoverPose(E,src_pts_norm,dst_pts_norm)

		#compare this tf to gazebo reported tf. any way to use these as estimates?
		M_src = np.hstack((R,t))
		M_dst = np.hstack((np.eye(3,3),np.zeros((3,1))))

		P_src = np.dot(K,M_src)
		P_dst = np.dot(K,M_dst)

		points3d = cv2.triangulatePoints(P_src,P_dst,src_pts,dst_pts)
		#points3d = cv2.triangulatePoints(projMat1,projMat2,src_pts,dst_pts)
		points3d /= points3d[3]
		points_world = np.dot((np.linalg.inv(tfMat1)), points3d)[:3,:]
		return points_world
	else:
		return 0

def populatePointCloud(points):
	header = Header()
	#header.seq = g_seq
	#g_seq += 1

	header.frame_id = 'map'
	header.stamp = rospy.Time.now()
	cloud = pc2.create_cloud_xyz32(header,points)

	return cloud

def main():
	global points
	rospy.init_node('recon', anonymous=True)
	pub = rospy.Publisher("reconstructed_pts", PointCloud2, queue_size = 1)
	img1 = cv2.imread('image_1.png')
	img2 = cv2.imread('image_2.png')
	pose1 = Pose()
	pose2 = Pose()
	pose1.position.x = 0.433012783527
	pose1.position.y = -0.249999880791
	pose1.position.z = 0.300000011921
	pose1.orientation.w = -0.226004580276
	pose1.orientation.x = 0.224733660268
	pose1.orientation.y = -0.0251856589319
	pose1.orientation.z = -0.947513796322

	pose2.position.x = -0.469846427441
	pose2.position.y = 0.171009778976
	pose2.position.z = 0.300000011921
	pose2.orientation.w = -0.955021155296
	pose2.orientation.x = -0.0593451678968
	pose2.orientation.y = -0.213517445104
	pose2.orientation.z = 0.197035643023

	tfmat1 = pose_to_tf(pose1)
	tfmat2 = pose_to_tf(pose2)

	while not rospy.is_shutdown():
		pts = make3d(img1,tfmat1,img2,tfmat2)
		pts = np.asarray(pts,np.float32)
		for i in range(pts.shape[1]):
			points.append([pts[0,i],pts[1,i],pts[2,i]])
		cloud = populatePointCloud(points)
		pub.publish(cloud)
		rospy.sleep(0.5)

if __name__ == '__main__':
	main()


# Calculate KZIT (index of first interior grid cell)
# interior grid cell = cell with no adjacent sidewall boundaries.
#
# Minimum of the 8 adjacent T-cells and the current T cell, -1 level.
# Author:      André Palóczy
# E-mail:      paloczy@gmail.com
# Date:        May/2019

import numpy as np
from netCDF4 import Dataset
from xarray import DataArray
from xarray import Dataset as Datasetx

##
continent_flag = -1
fname = "/lustre/atlas1/cli115/proj-shared/ia_top_tx0.1_v2_60yrs/ia_top_tx0.1_v2_yel_patc_1948_intel_def_year_2009/ocn/hist/ia_top_tx0.1_v2_yel_patc_1948_intel.pop.h.2009-01.nc"

nc = Dataset(fname)

lont = nc.variables['TLONG'][:]
latt = nc.variables['TLAT'][:]
z_t = nc.variables['z_t'][:]
h_t = nc.variables['HT'][:]
kmt = nc.variables['KMT'][:] # k index of deepest T-cell.

ny, nx = h_t.shape
nym = ny - 1
nxm = nx - 1
nz = z_t.size
kmt = kmt - 1 # Python indexing starts at 0, FORTRAN indexing starts at 1.

# Get index of deepest interior grid cell.
print("Calculating KZIT.")
kzit = np.zeros((ny,nx))

for j in range(ny):
    print(j+1," / ",ny)
    for i in range(nx):
        if kmt[j,i]==-1: # Continent mask.
            kzit[j,i] = -1
            continue
        else:
            if np.logical_and(j==0, i==0):
                kzit[j,i] = np.min([kmt[j,i-1], kmt[j+1,i-1], kmt[j+1,i], kmt[j+1,i+1], kmt[j,i+1], kmt[j,i]]) - 1
            elif np.logical_and(j==nym, i==0):
                kzit[j,i] = np.min([kmt[j-1,i-1], kmt[j,i-1], kmt[j,i+1], kmt[j-1,i+1], kmt[j-1,i], kmt[j,i]]) - 1
            elif np.logical_and(j==nym, i==nxm):
                kzit[j,i] = np.min([kmt[j-1,i-1], kmt[j,i-1], kmt[j,0], kmt[j-1,0], kmt[j-1,i], kmt[j,i]]) - 1
            elif np.logical_and(j==0, i==nxm):
                kzit[j,i] = np.min([kmt[j,i-1], kmt[j+1,i-1], kmt[j+1,i], kmt[j+1,0], kmt[j,0], kmt[j,i]]) - 1
            elif j==0:
                kzit[j,i] = np.min([kmt[j,i-1], kmt[j+1,i-1], kmt[j+1,i], kmt[j+1,i+1], kmt[j,i+1], kmt[j,i]]) - 1
            elif j==nym:
                kzit[j,i] = np.min([kmt[j-1,i-1], kmt[j,i-1], kmt[j,i+1], kmt[j-1,i+1], kmt[j-1,i], kmt[j,i]]) - 1
            elif i==0:
                kzit[j,i] = np.min([kmt[j-1,i-1], kmt[j,i-1], kmt[j+1,i-1], kmt[j+1,i], kmt[j+1,i+1], kmt[j,i+1], kmt[j-1,i+1], kmt[j-1,i], kmt[j,i]]) - 1
            elif i==nxm:
                kzit[j,i] = np.min([kmt[j-1,i-1], kmt[j,i-1], kmt[j+1,i-1], kmt[j+1,i], kmt[j+1,0], kmt[j,0], kmt[j-1,0], kmt[j-1,i], kmt[j,i]]) - 1
            else:
                kzit[j,i] = np.min([kmt[j-1,i-1], kmt[j,i-1], kmt[j+1,i-1], kmt[j+1,i], kmt[j+1,i+1], kmt[j,i+1], kmt[j-1,i+1], kmt[j-1,i], kmt[j,i]]) - 1

# Change back to fortran indexing. Continent mask is now 0.
kzit = np.int32(kzit + 1)
kmt = np.int32(kmt + 1)

kzit[kzit==-1] = 0

dims = ['x', 'y']
coords = dict(lont=(dims, lont), latt=(dims, latt))
kmt = DataArray(kmt, coords=coords, dims=dims)
kzit = DataArray(kzit, coords=coords, dims=dims)

fout = '/ccs/home/apaloczy/analysis/data/kzit.nc'
Datasetx(data_vars=dict(kmt=kmt, kzit=kzit), coords=coords).to_netcdf(fout)