#!/usr/bin/env python
# coding: utf-8

# In[3]:

import os
import glob
import sys

if len(sys.argv) < 2:
    print ("Please enter the path to the data directory")

f = {}
for root, directories, filenames in os.walk(sys.argv[1]): #Change to Data_MPHI for MPHI dataset
    for directory in directories:
        f[os.path.join(root, directory)] = {}
    for filename in filenames:
        f[root][os.path.splitext(filename)[0]] = os.path.join(root,filename)
print (f)

# In[4]:

import csv

lexicon = {}
with open('./MPQA/lexicon_easy.csv',"r") as csvfile:
    data = csv.reader(csvfile)
    for row in data:
        row[1] = int(row[1])
        row[2] = int(row[2])
        lexicon[row[0]] = {}
        lexicon[row[0]]['subj'] = row[1]
        lexicon[row[0]]['sent'] = row[2]

# In[5]:

# In[6]:

from nltk.corpus import wordnet as wn
from nltk.stem import WordNetLemmatizer
from nltk.corpus import sentiwordnet as swn
from nltk import sent_tokenize, word_tokenize, pos_tag
from nltk import ngrams,bigrams

def mpqa_subj(tweet):
    feat = 0
    score = 0
    tokens = word_tokenize(tweet)
    for token in tokens:
        if token in lexicon:
            score+= lexicon [token]['subj']
    if score > 2 or score < -2:
        feat = 1
    return feat

def pot_adj(tweet):
    feat = 0
    tokens = word_tokenize(tweet)
    for token in tokens:
        synsets = wn.synsets(token)
        for s in synsets:
            if s.pos() == 'a':
                feat = 1
    return feat

# In[7]:

lemmatizer = WordNetLemmatizer()

def penn_to_wn(tag):
    """
    Convert between the PennTreebank tags to simple Wordnet tags
    """
    if tag.startswith('J'):
        return wn.ADJ
    elif tag.startswith('N'):
        return wn.NOUN
    elif tag.startswith('R'):
        return wn.ADV
    elif tag.startswith('V'):
        return wn.VERB
    return None

def swn_polarity(tweet):
    """
    Return a sentiment polarity: 0 = negative, 1 = positive
    """

    sentiment = 0.0
    tokens_count = 0

    tagged_sentence = pos_tag(word_tokenize(tweet))

    for word, tag in tagged_sentence:
        wn_tag = penn_to_wn(tag)
        if wn_tag not in (wn.NOUN, wn.ADJ, wn.ADV):
            continue

        lemma = lemmatizer.lemmatize(word, pos=wn_tag)
        if not lemma:
            continue

        synsets = wn.synsets(lemma, pos=wn_tag)
        if not synsets:
            continue

        # Take the first sense, the most common
        synset = synsets[0]
        swn_synset = swn.senti_synset(synset.name())

        if swn_synset.pos_score() - swn_synset.neg_score()>0:
            sentiment+=1
        elif swn_synset.pos_score() - swn_synset.neg_score()<0:
            sentiment+=-1
        tokens_count += 1

    # judgment call ? Default to positive or negative
    if not tokens_count:
        return 0

    return sentiment

def sentiword_mpqa_sentiment(tweet):
    feat = [0,0]
    feat[0] = swn_polarity(tweet)
    tokens = word_tokenize(tweet)
    for token in tokens:
        if token in lexicon:
            feat[1]+= lexicon[token]['sent']
    return feat

def target_yn(tweet,target):
    if target in tweet:
        return 1
    else:
        return 0

def word_ngrams(tweet,target):
    feat= [0,0,0]
    tweet_sent = tweet.split()
    target_sent =target.split()
    for k in range(1,4):
        for target_gram in ngrams(target_sent,k):
            for tweet_gram in ngrams(tweet_sent,k):
                if tweet_gram == target_gram:
                    feat[k-1] = 1
    return feat

def char_ngrams(tweet,target):
    feat= [0,0,0]
    tweet_sent = ''.join(e for e in tweet if e.isalnum())
    target_sent = ''.join(e for e in target if e.isalnum())
    for k in range(2,5):
        for target_gram in ngrams(target_sent,k):
            for tweet_gram in ngrams(tweet_sent,k):
                if tweet_gram == target_gram:
                    feat[k-2] = 1
    return feat

# In[8]:

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer

data = {}
for key in sorted(f.keys()):
    features = {}

    #Setting up the file names
    train_file = f[key]['train_clean']
    test_file = f[key]['test_clean']

    #Set up the TF-IDF Vectorizer
    corpus = []
    with open(train_file,'r') as fr:
        lines = fr.readlines()
        for line in lines:
            row = line.split('\t')
            if row[0] == 'ID':
                continue
            tweet = row[2].lower()
            corpus.append(tweet)
    with open(test_file,'r') as fr:
        lines = fr.readlines()
        for line in lines:
            row = line.split('\t')
            if row[0] == 'ID':
                continue
            tweet = row[2].lower()
            corpus.append(tweet)
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(corpus)
    X = X.todense()

    #Train Feature Extraction
    train_ph1 = []
    train_ph2 = []
    i = 0
    with open(train_file,'r') as fr:
        lines = fr.readlines()
        for line in lines:
            feature_vector_1 = []
            feature_vector_2 = []
            row = line.split('\t')
            if row[0] == 'ID':
                continue
            tweet = row[2].lower()
            target = row[1].lower()

            #First phase features
            feature_vector_1.append(mpqa_subj(tweet))
            feature_vector_1.append(pot_adj(tweet))
            #Uncomment the following line to add count vectorizer
            #feature_vector_1.extend(X[i].tolist()[0])

            #Second Phase features
            senti = sentiword_mpqa_sentiment(tweet)
            feature_vector_2.extend(senti)
            feature_vector_2.append(target_yn(tweet,target))
            feature_vector_2.extend(word_ngrams(tweet,target))
            feature_vector_2.extend(char_ngrams(tweet,target))
            #Uncomment the following line to add count vectorizer
            #feature_vector_2.extend(X[i].tolist()[0])

            i+=1

            category = row[3].rstrip()
#             print (category)
            if category == 'NONE':
                feature_vector_1.append(1)
                train_ph1.append(feature_vector_1)
            else:
                feature_vector_1.append(0)
                train_ph1.append(feature_vector_1)
                if category == 'AGAINST':
                    feature_vector_2.append(0)
                    train_ph2.append(feature_vector_2)
                else:
                    feature_vector_2.append(1)
                    train_ph2.append(feature_vector_2)

    #Load the test data and calculate features
    test = []
    test_y = []
    with open(test_file,'r') as fr:
        lines = fr.readlines()
        for line in lines:
            feature_vector_1 = []
            feature_vector_2 = []
            row = line.split('\t')
            if row[0] == 'ID':
                continue
            tweet = row[2].lower()
            target = row[1].lower()

            #First phase features
            feature_vector_1.append(mpqa_subj(tweet))
            feature_vector_1.append(pot_adj(tweet))
            #Uncomment the following line to add count vectorizer
            #feature_vector_1.extend(X[i].tolist()[0])

            #Second Phase features
            senti = sentiword_mpqa_sentiment(tweet)
            feature_vector_2.extend(senti)
            feature_vector_2.append(target_yn(tweet,target))
            feature_vector_2.extend(word_ngrams(tweet,target))
            feature_vector_2.extend(char_ngrams(tweet,target))
            #Uncomment the following line to add count vectorizer
            #feature_vector_2.extend(X[i].tolist()[0])

            i+=1

            test.append((np.array(feature_vector_1,dtype=np.int32),np.array(feature_vector_2,dtype=np.int32)))

            category = row[3].rstrip()
            if category == 'NONE':
                test_y.append(2)
            else:
                if category == 'AGAINST':
                    test_y.append(1)
                else:
                    test_y.append(0)
#     print (train_ph1[0])
    train_ph1 = np.array(train_ph1, dtype = np.int32)
    train_ph2 = np.array(train_ph2,dtype = np.int32)
#     test = np.array(test,dtype = np.int32)
    test_y = np.array(test_y,dtype = np.int32)
    print (key)
    print (train_ph1.shape)
    print (train_ph2.shape)
    print (test_y.shape)
    data[key] = (train_ph1,train_ph2,test,test_y)
    print ()

# In[9]:

from sklearn.svm import LinearSVC
from sklearn.utils import shuffle

def predict(clf1,clf2,x):
    ph1 = [x[0]]
#     print (ph1)
    ph2 = [x[1]]
#     print (ph2)
    p1 = clf1.predict(ph1)
#     print (p1)
    if p1[0] == 1:
        return 2
    else:
        p2 = clf2.predict(ph2)
        if p2[0] == 1:
            return 0
        else:
            return 1

def score(y_true,y_pred):
    fav = [0,0,0]
    ag = [0,0,0]
    tot = [fav,ag]
    corr = 0
    for y_t,y_p in zip(y_true,y_pred):
        if y_t < 2:
            tot[y_t][2]+=1
        if y_p < 2:
            tot[y_p][1]+=1
        if y_t == y_p and y_t < 2:
            tot[y_t][0]+=1
        if y_t == y_p:
            corr+=1

        r0 = tot[0][0]/(tot[0][2]+1e-5)
        p0 = tot[0][0]/(tot[0][1]+1e-5)
        r1 = tot[1][0]/(tot[1][2]+1e-5)
        p1 = tot[1][0]/(tot[1][1]+1e-5)
        f0 = 2*r0*p0/(r0+p0+1e-5)
        f1 = 2*r1*p1/(r1+p1+1e-5)

        f_avg = (f0+f1)/2
    return tot,f_avg

# In[ ]:

import numpy as np
from sklearn.model_selection import GridSearchCV

parameters = {
    'C' : np.logspace(start = 0.001,stop = 5,num = 10),
    'dual' : [True, False]
}
tot = np.array([[0,0,0],[0,0,0]])
for key in data.keys():
    train_ph1 = shuffle(data[key][0])
    train_ph2 =shuffle(data[key][1])
    test = data[key][2]
    test_y = data[key][3]

    #Phase 1 Training
    train_ph1_x = train_ph1[:,:-1]
    train_ph1_y = train_ph1[:,-1]
#     print(train_ph1_x)
#     print(train_ph1_y)
    svc = LinearSVC(max_iter = 100000)
    clf1  = GridSearchCV(svc, parameters, cv=5)
    clf1.fit(train_ph1_x,train_ph1_y)

    #Phase 2 Training
    train_ph2_x = train_ph2[:,:-1]
    train_ph2_y = train_ph2[:,-1]

    clf2 = GridSearchCV(svc, parameters, cv=5)
    clf2.fit(train_ph2_x,train_ph2_y)

    #Testing
    preds = []
    for x in test:
        preds.append(predict(clf1,clf2,x))
#     print (preds)
#     print (test_y)
    print (key+":")
    arr, sc = score(test_y,preds)
    print ("Score - %f"%sc)
    print ()
    tot = tot + np.array(arr)
    cat = key.replace(sys.argv[1],"")
    with open(key+'/predict_'+cat+'.txt','w') as pf:
        for pred in preds:
            if pred == 0:
                pf.write("FAVOR\n")
            elif pred == 1:
                pf.write("AGAINST\n")
            elif pred == 2:
                pf.write("NONE\n")

# In[ ]:

r0 = tot[0][0]/(tot[0][2]+1e-5)
p0 = tot[0][0]/(tot[0][1]+1e-5)
r1 = tot[1][0]/(tot[1][2]+1e-5)
p1 = tot[1][0]/(tot[1][1]+1e-5)
f0 = 2*r0*p0/(r0+p0+1e-5)
f1 = 2*r1*p1/(r1+p1+1e-5)

f_avg = (f0+f1)/2
print ("Overall Score - %f"%f_avg)

# In[ ]:


#!/usr/bin/env python3
"""Parses the wikipedia pages and other sources into JSON."""

import json
import os
import re
from collections import namedtuple, OrderedDict

import iso3166
from bs4 import BeautifulSoup

MNC_OPERATOR_FIELDS = ("mcc", "mnc", "brand", "operator")
COUNTRY_FIELDS = ("name", "iso2", "iso3", "numeric", "mccs")
MNC_OPERATOR_FIELDS_ISO = ("mcc", "mnc", "brand", "operator", "iso")
MNC_FIELDS_ITU = ("mcc", "mnc", "brand", "country")
MNCOperatorISO = namedtuple("MNCOperatorISO", MNC_OPERATOR_FIELDS_ISO)
MNCOperatorITU = namedtuple("MNCOperatorITU", MNC_FIELDS_ITU)
MNCOperator = namedtuple("MNCOperator", MNC_OPERATOR_FIELDS)
CountriesD = namedtuple("CountryD", COUNTRY_FIELDS)
_CACHE = {}

def _load_json(cache_key, json_path, wrapper):
    global _CACHE
    if cache_key not in _CACHE:
        with open(json_path, "rb") as json_file:
            data = json.loads(json_file.read().decode())
            _CACHE[cache_key] = [wrapper(*line) for line in data]
    return _CACHE[cache_key]

def parse_wikipedia(source_dir):
    operators = []

    i = 0
    with open(os.path.join(source_dir, "wiki_" + str(i)), "r") as infile:
        htmlfile = infile.read()
    while i <= 10:
        fpath = os.path.join(source_dir, "wiki_" + str(i))
        if os.path.isfile(fpath):
            with open(fpath, "r") as infile:
                htmlfile = infile.read()
                soup = BeautifulSoup(htmlfile, "html.parser")
                for table in soup.find_all(
                    "table", class_="wikitable", attrs={"width": "100%"}
                ):
                    hs = table.find_previous_sibling("h4")
                    if hs is None:
                        hs = table.find_previous_sibling("h3")
                    iso = ""
                    if hs is not None:
                        hs = hs.find("span", class_="mw-headline")
                        if hs is not None:
                            if hs.a is not None:
                                i_tag = hs.a
                                i_tag.decompose()
                            iso = re.sub(
                                r"\(.*\)",
                                "",
                                hs.text.strip()
                                .strip("\n")
                                .replace("\n", "")
                                .replace("–", "")
                                .replace(" ", ""),
                            )
                            if iso == "GE-AB":
                                iso = "GE"
                            if iso == "British Indian Ocean Territory":
                                iso = "IO"
                            if "-" in iso:
                                print(
                                    "Another iso with region code, please check !: "
                                    + iso
                                )
                        prev = table.find_previous_sibling()
                        for li in prev.find_all("li"):
                            if li.a is not None:
                                i_tag = li.a
                                i_tag.decompose()
                            iso_ = re.sub(
                                r".* – ",
                                "",
                                li.text.strip().strip("\n").replace("\n", ""),
                            )
                            if iso_ == "GE-AB":
                                iso_ = "GE"
                            if iso_ == "British Indian Ocean Territory":
                                iso_ = "IO"
                            if "-" in iso_:
                                print(
                                    "Another iso with region code, please check !: "
                                    + iso_
                                )
                            if iso_ not in iso:
                                iso += "/" + iso_
                        if "IT" in iso:
                            iso_ = "VA"
                            if iso_ not in iso:
                                iso += "/" + iso_
                            iso_ = "SM"
                            if iso_ not in iso:
                                iso += "/" + iso_
                        if "FI" in iso:
                            iso_ = "AX"
                            if iso_ not in iso:
                                iso += "/" + iso_
                        if "MA" in iso:
                            iso_ = "EH"
                            if iso_ not in iso:
                                iso += "/" + iso_
                    for row in table.find_all("tr"):
                        mcc, mnc, brand, operator = row.find_all_next("td", limit=4)
                        if mcc.text in ["MCC", ""]:
                            continue
                        if mcc.div is not None:
                            i_tag = mcc.div
                            i_tag.decompose()
                        if mcc.span is not None:
                            i_tag = mcc.span
                            i_tag.decompose()
                        mcc = mcc.text.strip().strip("\n").replace("\n", "")
                        if mnc.div is not None:
                            i_tag = mnc.div
                            i_tag.decompose()
                        if mnc.span is not None:
                            i_tag = mnc.span
                            i_tag.decompose()
                        mnc = (
                            mnc.text.strip()
                            .strip("\n")
                            .replace("\n", "")
                            .replace(" ", "")
                        )
                        if operator.div is not None:
                            i_tag = operator.div
                            i_tag.decompose()
                        if operator.span is not None:
                            i_tag = operator.span
                            i_tag.decompose()
                        operator = operator.text.strip().strip("\n").replace("\n", "")
                        if brand.div is not None:
                            i_tag = brand.div
                            i_tag.decompose()
                        if brand.span is not None:
                            i_tag = brand.span
                            i_tag.decompose()
                        brand = brand.text.strip().strip("\n").replace("\n", "")
                        if "-" in mnc:
                            mncb = mnc.split("-")[0]
                            mncb_len = len(mncb)
                            mnce = mnc.split("-")[1]
                            mnce_len = len(mnce)
                            if mncb_len == mnce_len:
                                for mnc_ in range(int(mncb), int(mnce)):
                                    mnc = "%0*d" % (mncb_len, mnc_)
                                    operators.append(
                                        MNCOperatorISO(
                                            mcc=mcc,
                                            mnc=mnc,
                                            brand=brand,
                                            operator=operator,
                                            iso=iso,
                                        )
                                    )
                                # because range does not include the stop element, we add it here
                                operators.append(
                                    MNCOperatorISO(
                                        mcc=mcc,
                                        mnc=mnce,
                                        brand=brand,
                                        operator=operator,
                                        iso=iso,
                                    )
                                )
                            else:
                                print("don't know how to deal with this range: " + mnc)
                                operators.append(
                                    MNCOperatorISO(
                                        mcc=mcc,
                                        mnc=mnc,
                                        brand=brand,
                                        operator=operator,
                                        iso=iso,
                                    )
                                )
                        else:
                            operators.append(
                                MNCOperatorISO(
                                    mcc=mcc,
                                    mnc=mnc,
                                    brand=brand,
                                    operator=operator,
                                    iso=iso,
                                )
                            )

        else:
            break
        i += 1

    return operators

def parse_itu(source_dir):
    # The itu.json file was created from the English word document: https://www.itu.int/dms_pub/itu-t/opb/sp/T-SP-E.212B-2018-MSW-E.docx
    return _load_json("itu.json", os.path.join(source_dir, "itu.json"), MNCOperatorITU)

def parse_mcc_mnc_table(source_dir):
    with open(os.path.join(source_dir, "mcc-mnc-table.json"), "r") as infile:
        htmlfile = infile.read()
    if len(htmlfile) > 0:
        return json.loads(htmlfile)
    else:
        return {}

def merge_wiki_itu(source_dir, json_dir):
    wiki_operators = parse_wikipedia(source_dir)
    itu_operators = parse_itu(source_dir)
    mcc_mnc_operators = parse_mcc_mnc_table(source_dir)
    merged_operators = {}
    sorted_operators = OrderedDict()
    country_dict = {}
    country_list = iso3166.countries
    countries_sorted = OrderedDict()
    for country in country_list:
        country_dict[country.alpha2] = CountriesD._make(
            [country.name, country.alpha2, country.alpha3, country.numeric, None]
        )

    for operator_itu in itu_operators:
        cname = operator_itu.country
        # rename the country names to match them with the ones from iso3166
        cname = cname.replace("*", "")
        cname = cname.replace("Czech Rep.", "Czechia")
        cname = cname.replace("Rep.", "Republic")
        cname = cname.replace(
            "United Kingdom", "United Kingdom of Great Britain and Northern Ireland"
        )
        cname = cname.replace(" (", ", ").replace(")", "")
        cname = cname.replace(
            "The Former Yugoslav Republic of Macedonia", "North Macedonia"
        )
        cname = cname.replace("United States", "United States of America")
        cname = cname.replace("British Virgin Islands", "Virgin Islands, British")
        cname = cname.replace(
            "Dem. Republic of the Congo", "Congo, Democratic Republic of the"
        )
        cname = cname.replace("Tanzania", "Tanzania, United Republic of")
        cname = cname.replace("Hong Kong, China", "Hong Kong")
        cname = cname.replace("Macao, China", "Macao")
        cname = cname.replace("Lao P.D.R.", "Lao People's Democratic Republic")
        cname = cname.replace("Micronesia", "Micronesia, Federated States of")
        cname = cname.replace(
            "French Departments and Territories in the Indian Ocean",
            "French Southern Territories",
        )
        cname = cname.replace(
            "Falkland Islands, Malvinas", "Falkland Islands (Malvinas)"
        )
        country = None
        if cname != "":
            country = country_list.get(cname)
        if country is not None:
            iso = country.alpha2
            if country_dict[iso] is not None:
                c = country_dict[iso]
                if c.mccs is None:
                    c = c._replace(mccs=operator_itu.mcc)
                else:
                    if operator_itu.mcc not in c.mccs:
                        if isinstance(c.mccs, list):
                            xmcc = list(c.mccs)
                        else:
                            xmcc = [c.mccs]
                        xmcc.append(operator_itu.mcc)
                        c = c._replace(mccs=xmcc)
                country_dict[iso] = c
        operator = MNCOperator._make(
            [operator_itu.mcc, operator_itu.mnc, operator_itu.brand, operator_itu.brand]
        )
        operator_key = operator.mcc, operator.mnc
        merged_operators[operator_key] = list(operator._asdict().values())

    for operator_mcc_mnc in mcc_mnc_operators:
        iso = operator_mcc_mnc["iso"].upper()
        # fix some iso2 country codes
        if iso == "FG":
            iso = "GF"
        if iso == "AN":
            iso = "BQ"
        if iso == "TP":
            iso = "TL"
        if iso == "TK" and operator_mcc_mnc["country"].upper() == "TAJIKISTAN":
            iso = "TJ"
        if iso == "VI" and operator_mcc_mnc["mcc"] == "376":
            iso = "TC"
        if iso == "N/A":
            iso = None
        if iso is not None:
            if country_dict[iso] is not None:
                c = country_dict[iso]
                if c.mccs is None:
                    c = c._replace(mccs=operator_mcc_mnc["mcc"])
                else:
                    if operator_mcc_mnc["mcc"] not in c.mccs:
                        if isinstance(c.mccs, list):
                            xmcc = list(c.mccs)
                        else:
                            xmcc = [c.mccs]
                        xmcc.append(operator_mcc_mnc["mcc"])
                        c = c._replace(mccs=xmcc)
                country_dict[iso] = c
        operator = MNCOperator._make(
            [
                operator_mcc_mnc["mcc"],
                operator_mcc_mnc["mnc"],
                operator_mcc_mnc["network"],
                operator_mcc_mnc["network"],
            ]
        )
        operator_key = operator.mcc, operator.mnc
        merged_operators[operator_key] = list(operator._asdict().values())

    for operator_iso in wiki_operators:
        if operator_iso.iso is not None and operator_iso.iso != "":
            isos = operator_iso.iso.split("/")
            if isos is None:
                print("fallback")
                isos = iso
            for iso in isos:
                if country_dict[iso] is not None:
                    c = country_dict[iso]
                    if c.mccs is None:
                        c = c._replace(mccs=operator_iso.mcc)
                    else:
                        if operator_iso.mcc not in c.mccs:
                            if isinstance(c.mccs, list):
                                xmcc = list(c.mccs)
                            else:
                                xmcc = [c.mccs]
                            xmcc.append(operator_iso.mcc)
                            c = c._replace(mccs=xmcc)
                    country_dict[iso] = c
        operator = MNCOperator._make(
            [
                operator_iso.mcc,
                operator_iso.mnc,
                operator_iso.brand,
                operator_iso.operator,
            ]
        )
        operator_key = operator.mcc, operator.mnc
        merged_operators[operator_key] = list(operator._asdict().values())

    for key, value in sorted(country_dict.items(), key=lambda t: t[1]):
        x_list = list(value._asdict().values())
        if isinstance(x_list[4], list):
            x_sorted = sorted(x_list[4])
        else:
            x_sorted = x_list[4]
        x_list[4] = x_sorted
        countries_sorted[key] = x_list
    with open(os.path.join(json_dir, "countries.json"), "w") as outfile:
        json_string = json.dumps(list(countries_sorted.values()), ensure_ascii=True)
        # Should be a better way to obtain the desired custom pretty formatted json file, maybe with a custom class ...
        json_string = re.sub(r"\], \[", "],\n    [", json_string)
        json_string = re.sub(r"\]\]$", "]\n]", json_string)
        json_string = re.sub(r"\[\[", "[\n    [", json_string) + "\n"
        outfile.write(json_string)
    for key, value in sorted(merged_operators.items()):
        sorted_operators[key] = value
    return list(sorted_operators.values())

def write_operators(operators, json_dir):
    with open(os.path.join(json_dir, "mnc_operators.json"), "w") as outfile:
        json_string = json.dumps(operators, ensure_ascii=True)
        # Should be a better way to obtain the desired custom pretty formatted json file, maybe with a custom class ...
        json_string = re.sub(r"\], \[", "],\n    [", json_string)
        json_string = re.sub(r"\]\]$", "]\n]", json_string)
        json_string = re.sub(r"\[\[", "[\n    [", json_string) + "\n"
        outfile.write(json_string)

if __name__ == "__main__":
    my_dir = os.path.dirname(os.path.abspath(__file__))
    download_dir = os.path.join(my_dir, "source_data")
    json_dir = os.path.join(my_dir, "json")
    data = merge_wiki_itu(download_dir, json_dir)
    write_operators(data, json_dir)


import mlrose_hiive as mlrose
import numpy as np
import matplotlib.pyplot as plt
import six
weights = np.random.randint(1, 50, 50)  # 100个物品，重量随机在1到50之间
values = np.random.randint(1, 50, 50)  # 100个物品，价值随机在5到100之间
max_weight_pct = 0.4
fitness1 = mlrose.Knapsack(weights, values, max_weight_pct)
problem1 = mlrose.DiscreteOpt(length=len(weights), fitness_fn=fitness1, maximize=True)
problem1.set_mimic_fast_mode(True)

# 问题1：FourPeaks，强调遗传算法的优点
import numpy as np
import mlrose_hiive
from mlrose_hiive import DiscreteOpt, random_hill_climb, simulated_annealing, genetic_alg, mimic
from mlrose_hiive.fitness import FourPeaks, OneMax, FlipFlop

# 问题2：TSP，强调退火算法的优点

coords_list = [(1, 1), (4, 2), (5, 2), (6, 4), (4, 4), (3, 6), (1, 5), (2, 3)]

# 定义问题
fitness2 = mlrose.TravellingSales(coords=coords_list)
problem2 = mlrose.TSPOpt(length=len(coords_list), fitness_fn=fitness2, maximize=False)

problem2.set_mimic_fast_mode(True)
# 为模拟退火定义调度（温度函数）
schedule = mlrose_hiive.GeomDecay(init_temp=10, decay=0.95, min_temp=0.001)

fitness3 = FlipFlop()
problem3 = DiscreteOpt(length=200, fitness_fn=fitness3)
problem3.set_mimic_fast_mode(True)
problems = [problem1,problem2,problem3]

algorithms = ["Random Hill Climb", "Simulated Annealing", "Genetic Algorithm", "MIMIC"]
problem_names = ["Knapsack","TSP", "FlipFlop"]

def apply_algorithms(problem):
    # 随机爬山
    _, best_fitness_rhc, _ = random_hill_climb(problem, restarts=10, max_attempts=50, max_iters=50, random_state=42)
    # 模拟退火
    _, best_fitness_sa, _ = simulated_annealing(problem, schedule=mlrose_hiive.GeomDecay(), max_attempts=50, max_iters=50, random_state=42)
    # 遗传算法
    _, best_fitness_ga, _ = genetic_alg(problem, pop_size=200, mutation_prob=0.1, max_attempts=50, max_iters=50, random_state=42)
    # MIMIC
    _, best_fitness_mimic, _ = mimic(problem, pop_size=200, keep_pct=0.2, max_attempts=50, max_iters=50, random_state=42)

    return best_fitness_rhc, best_fitness_sa, best_fitness_ga, best_fitness_mimic,

results = [apply_algorithms(p) for p in problems]

for i, r in enumerate(results):
    print(f"Problem: {problem_names[i]}")
    for j, alg in enumerate(algorithms):
        print(f"{alg}: {r[j]}")
    print("\n")

def apply_algorithms(problem):
    # 注意curve=True的设置
    best_state_rhc, best_fitness_rhc, fitness_curve_rhc = random_hill_climb(problem, restarts=100, max_attempts=50, max_iters=50, random_state=42, curve=True)
    best_state_sa, best_fitness_sa, fitness_curve_sa = simulated_annealing(problem, schedule=schedule, max_attempts=50, max_iters=50, random_state=42, curve=True)
    best_state_ga, best_fitness_ga, fitness_curve_ga = genetic_alg(problem, pop_size=200, mutation_prob=0.1, max_attempts=50, max_iters=50, random_state=42, curve=True)
    best_state_mimic, best_fitness_mimic, fitness_curve_mimic = mimic(problem, pop_size=200, keep_pct=0.2, max_attempts=50, max_iters=50, random_state=42, curve=True)

    return fitness_curve_rhc, fitness_curve_sa, fitness_curve_ga, fitness_curve_mimic

results = [apply_algorithms(p) for p in problems]

for i, curves in enumerate(results):
    plt.figure(figsize=(12, 8))

    for curve, alg in zip(curves, algorithms):
        # cv = []
        # for k in range(len(curve)):
        #     cv.append([k,curve[k][0]])
        print(alg)
        print(curve)

        plt.plot(range(len(curve)),curve[:,1], label=alg)

    plt.title(f"Wall Clock Time/ Iters {problem_names[i]}")
    plt.xlabel("Iterations")
    plt.ylabel("WCT")
    plt.legend()
    plt.grid(True)
    plt.show()
# for i, curves in enumerate(results):
#     plt.figure(figsize=(12, 8))

#     for curve, alg in zip(curves, algorithms):
#         plt.plot(range(len(curve)),curve[:,1], label=alg)

#     plt.title(f"Fitness Curve for {problem_names[i]}")
#     plt.xlabel("Iterations")
#     plt.ylabel("Fitness")
#     plt.legend()
#     plt.grid(True)
#     plt.show()


import os
import tempfile

import pytest
from pytest_splunk_addon.standard_lib.utilities.junit_parser import JunitParser

def test_parse_junit_raises_exception_when_not_fields_are_present_in_testcase():
    jp = JunitParser(
        os.path.join(
            os.path.dirname(__file__), "test_data", "sample_junit_without_datamodel.xml"
        )
    )
    with pytest.raises(Exception):
        jp.parse_junit()

def test_junit_parser_parser_junit():
    expected_result = [
        {
            "data_model": "datamodel_1",
            "fields": "field_1",
            "data_set": "dataset_1",
            "fields_type": "required",
            "tag_stanza": 'dataset_name="dataset_1"',
            "status": "failed",
            "test_property": "splunklib.binding.HTTPError: HTTP 400 Bad Request",
        },
        {
            "data_model": "datamodel_2",
            "fields": "field_2",
            "data_set": "dataset_2",
            "fields_type": "required",
            "tag_stanza": 'dataset_name="dataset_2"',
            "status": "failed",
            "test_property": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
        },
        {
            "data_model": "datamodel_3",
            "fields": "field_3",
            "data_set": "dataset_3",
            "fields_type": "conditional",
            "tag_stanza": 'dataset_name="dataset_3"',
            "status": "passed",
            "test_property": "-",
        },
        {
            "data_model": "datamodel_4",
            "fields": "field_4",
            "data_set": "dataset_4",
            "fields_type": "required",
            "tag_stanza": 'dataset_name="dataset_4"',
            "status": "skipped",
            "test_property": "-",
        },
    ]
    jp = JunitParser(
        os.path.join(os.path.dirname(__file__), "test_data", "sample_junit.xml")
    )
    jp.parse_junit()
    assert jp.data == expected_result

def test_junit_parser_init_with__directory():
    with tempfile.TemporaryDirectory() as tempdir:
        with pytest.raises(Exception):
            JunitParser(tempdir)

def test_junit_parser_init_with_not_found_file():
    with pytest.raises(Exception):
        JunitParser("file-does-not-exist")


import csv
import scipy.stats
import numpy as np

def mean_confidence_interval(data, confidence=0.95):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return m, m-h, m+h

all_surps_all_k_all_n = dict()

for k in range(1, 5):
    all_surps_all_n = dict()

    for n in range(1, 9):
        print(f'----------- new value of n = {n}, k = {k} -----------')

        data = []

        with open(f'../data/{k}/output/test-{n}.txt', 'r') as f:
            reader = csv.reader(f, delimiter = '\t')
            for line in reader:
                if line[0] != '<unk>' and line[0] != '':
                    data.append(float(line[1]))

        data = data[:(len(data) // 8 * 8)]

        all_surps = {0 : [], 1 : [], 2 : [], 3 : []}

        j = 0
        for i in range(0, len(data), 8):
            for j in range(0, 4):
                all_surps[j].append(data[i + 2 * j + 1] - data[i + 2 * j])

        all_surps_all_n[n] = [[x, *mean_confidence_interval(all_surps[x])] for x in all_surps]

    all_surps_all_k_all_n[k] = all_surps_all_n

with open('../analysis/lstm.csv', 'w') as f:
    writer = csv.writer(f, delimiter = '\t')
    writer.writerow(['n', 'k', 'theme', 'recipient', 'pref', 'lower', 'upper'])
    for k in all_surps_all_k_all_n:
        for j in all_surps_all_k_all_n[k]:
            item = all_surps_all_k_all_n[k][j]
            for i, lengths in enumerate([[x, y] for x in ['short', 'long'] for y in ['short', 'long']]):
                writer.writerow([f'n = {j}', f'k = {k}', lengths[0], lengths[1], item[i][1], item[i][2], item[i][3]])


import PyDAQmx
from daq9211 import *

import Tkinter as tk
import tkFileDialog
import tkMessageBox
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
from matplotlib import pyplot as plt
import matplotlib.animation as animation
import datetime
import ttk

class daq_config_frame(tk.Frame):
    def __init__(self, master, controller, daq9211):
        tk.Frame.__init__(self,master)
        self.master = master
        self.daq9211 = daq9211
        self.selectedchannel = tk.IntVar()
        self.selectedchannel.set(0)
        #keep track of raised tc or voltage frames
        self.selectedframes = [None, None, None, None]

        #self.grid_columnconfigure(0, weight=1)
        self.grid_columnconfigure(1, weight=1)
        self.grid_columnconfigure(2, weight=1)
        #self.grid_rowconfigure(1, weight = 1)
        self.grid_rowconfigure(3, weight = 1)
        self.label = tk.Label(self, text="DAQ Configuration", justify = tk.CENTER, font = ("tkDefaultFont",24))
        self.label.grid(row = 0, column = 0, columnspan = 2, sticky = 'ew')

        #add button frame to add and delete channels
        self.btnframe = tk.Frame(self)
        self.btnframe.grid(row = 1, column = 0, sticky = 'ne')
        self.selectedlbl_str = tk.StringVar()
        self.selectedlbl_str.set('Selected Channel: %d' % self.selectedchannel.get())
        self.selectedlbl = tk.Label(self.btnframe, textvariable = self.selectedlbl_str, font = ("tkDefaultFont",16))
        self.selectedlbl.grid(row = 0, column = 0, sticky = 'ew')
        self.addtcbtn = ttk.Button(self.btnframe, text = "Add Thermocouple", command = lambda: self.raise_frame(self.thermocouple_frames[self.selectedchannel.get()]))
        self.addtcbtn.grid(row = 1, column = 0, sticky = 'ew')
        self.addvolbtn = ttk.Button(self.btnframe, text = "Add Analog Voltage", command = lambda: self.raise_frame(self.voltage_frames[self.selectedchannel.get()]))
        self.addvolbtn.grid(row = 2, column = 0, sticky = 'ew')

        # ID frames - consisting of btn_ID_frames
        self.btn_ID_frame = tk.Frame(self,borderwidth = 5, relief = tk.GROOVE)
        self.btn_ID_frame.grid(row = 2, column = 0, sticky = 'new')
        for i in range (0,4):
            self.btn_ID_frame.grid_rowconfigure(i, weight = 1)
        self.btn_ID_frame.grid_columnconfigure(0, weight = 1)
        self.channelID_frames = []
        for i in range(0,4):
            self.channelID_frames.append(daq_channelID_frame(self.btn_ID_frame,controller,daq9211,i))
            self.channelID_frames[i].grid(row = i, column = 0, sticky = 'nsew')

        self.configbtn_frame = tk.Frame(self)
        self.configbtn_frame.grid_columnconfigure(0,weight = 1)
        self.configbtn_frame.grid(row = 3, column = 0, sticky = 'new')
        self.openconfig_btn = ttk.Button(self.configbtn_frame, text = "Open Config", command = lambda: self.open_config())
        self.openconfig_btn.grid(row = 0, column = 0, sticky = 'nsew')
        self.saveconfig_btn = ttk.Button(self.configbtn_frame, text = "Save Config", command = lambda: self.save_config())
        self.saveconfig_btn.grid(row = 1, column = 0, sticky = 'nsew')

        # tc and voltage frames
        self.mainchannelframe = tk.Frame(self, borderwidth = 5, relief = tk.GROOVE)
        self.mainchannelframe.grid(row = 1, column = 1, rowspan = 3, sticky = 'nsew')
        self.mainchannelframe.grid_columnconfigure(0, weight = 1)
        self.mainchannelframe.grid_rowconfigure(0, weight = 1)
        self.mainchannelframe.grid_rowconfigure(1, weight = 1)
        self.mainchannelframe.grid_rowconfigure(2, weight = 1)
        self.mainchannelframe.grid_rowconfigure(3, weight = 1)

        #tc channel frame
        self.thermocouple_frames = []
        for i in range(0,4):
            self.thermocouple_frames.append(daq_thermocouple_frame(self.mainchannelframe,controller,daq9211,i))
            self.thermocouple_frames[i].grid(row = i, column = 0, sticky = 'nsew')

        #voltage channel frame
        self.voltage_frames = []
        for i in range(0,4):
            self.voltage_frames.append(daq_voltage_frame(self.mainchannelframe,controller,daq9211,i))
            self.voltage_frames[i].grid(row = i, column = 0, sticky = 'nsew')

        #raise the click frames
        for frame in self.channelID_frames:
            frame.tkraise()

        #add frame to record configuration of daq instance
        self.config_frame = tk.Frame(self, borderwidth = 5, relief = tk.GROOVE)
        self.config_frame.grid(row = 1, column = 2, rowspan = 3, sticky = 'nsew')
        self.config_frame.grid_columnconfigure(0, weight = 1)
        self.config_frame.grid_columnconfigure(1, weight = 1)
        self.config_frame.grid_rowconfigure(1, weight = 1)
        #list box to show configureations
        self.config_lbl = tk.Label(self.config_frame, text = "Configured Channels", font = ("tkDefaultFont", 14))
        self.config_lbl.grid(row = 0, column = 0, columnspan = 2, sticky = 'new')
        self.config_listbox = tk.Listbox(self.config_frame)
        self.config_listbox.grid(row = 1, rowspan = 2, column = 0, columnspan = 2, sticky = 'nsew')
      #  self.setuptask_btn = ttk.Button(self.config_frame, text = 'Setup Channel Tasks', command = lambda: self.run_tasksetup())
       # self.setuptask_btn.grid(row = 3, column = 0, columnspan = 2, sticky = 'ew')

    def raise_frame(self, frame):
        self.selectedframes[self.selectedchannel.get()] = frame
        frame.tkraise()

    def open_config(self):
        f = tkFileDialog.askopenfile(initialdir = 'daq_config', defaultextension = '.dat')

        for line in f:
            if "Channel" in line:
                #get the channel ID
                channel_ID =  int(line.strip('Channel: ').strip('\n'))
                self.selectedchannel.set(channel_ID)
            if "Sensor" in line:
                if "Thermocouple" in line:
                    selected_frame = self.thermocouple_frames[self.selectedchannel.get()]
                    selected_frame.ID = channel_ID
                    self.raise_frame(selected_frame)
                elif "Voltage" in line:
                    selected_frame = voltage_frames[self.selectedchannel.get()]
                    selected_frame.ID = channel_ID
                    self.raise_frame(selected_frame)
                else:
                    selected_frame = None
            if "Type" in line:
                selected_frame.type_str.set(line.strip('Type: ').strip('\n'))
            if "Min" in line:
                selected_frame.min_str.set(line.strip('Min: ').strip('\n'))
            if "Max" in line:
                selected_frame.max_str.set(line.strip('Max: ').strip('\n'))
                #self.selectedchannel.set()#set the selected channel to be the one from the file
            if "CJC" in line:
                selected_frame.cjc_str.set(line.strip('CJC: ').strip('\n'))

    def save_config(self):
        #get configs from each of the 4 channel frames
        today = datetime.date.today().strftime('%Y%m%d')
        config_name = 'daq9211_%s' % today
        f = tkFileDialog.asksaveasfile(mode = 'w', initialdir = 'daq_config', initialfile = config_name, defaultextension = '.dat')
        for i in range(0,4):
            data = self.get_channel_info(i)
            if data is not None:
                if data['sensor'] == 'Thermocouple':
                    #write thermocouple config_str
                    config_str = 'Channel: %d\nSensor: Thermocouple\nType: %s\nMin: %s\nMax: %s\nCJC: %s\n' % (data['ID'], data['type'], data['min'], data['max'], data['cjc'])
                    f.write(config_str)
                elif data['sensor'] == 'Voltage':
                    #write voltage config_str
                    config_str = 'Channel: %d\n, Sensor: Voltage\n' % (data['ID'])
                    f.write(config_str)
                else:
                    config_str = ""

    def get_channel_info(self,ID):
        frame = self.selectedframes[ID]
        data = {}
        if isinstance(frame, daq_thermocouple_frame):
            data['sensor'] = 'Thermocouple'
            data['ID'] = frame.ID
            data['type'] = frame.type_str.get()
            data['min'] = frame.min_str.get()
            data['max'] = frame.max_str.get()
            data['cjc'] = frame.cjc_str.get()
            return data
        elif isinstance(frame,daq_voltage_frame):
            data['sensor'] = 'Voltage'
            data['ID'] = frame.ID

        else:
            return None

    def run_tasksetup(self):
        for i in range(0,len(self.daq9211.channels)):
            self.daq9211.channels[i].setup_task()

class daq_channelID_frame(tk.Frame):
    def __init__(self, master, controller, daq9211, ID):
        tk.Frame.__init__(self,master)
        self.config(borderwidth = 5, relief = tk.GROOVE)
        self.grid_rowconfigure(0, weight = 1)
        self.grid_columnconfigure(0, weight = 1)
        self.ID = ID
        self.btn = tk.Button(self,text = '%d' % self.ID, font = ("tkDefaultFont",16), command = lambda: self.update_selectedchannel())
        self.btn.grid(row = 0, column = 0, sticky = 'nsew')
    def update_selectedchannel(self):
        self.master.master.selectedchannel.set(self.ID)
        self.master.master.selectedlbl_str.set('Selected Channel: %d' % self.master.master.selectedchannel.get())

class daq_thermocouple_frame(tk.Frame):
    def __init__(self, master, controller, daq9211, ID):
        tk.Frame.__init__(self, master)
        self.config(borderwidth = 5, relief = tk.GROOVE)
        self.grid_rowconfigure(7, weight = 1)
        self.grid_columnconfigure(0, weight = 1)
        self.grid_columnconfigure(1, weight = 1)
       # self.grid_columnconfigure(2, weight = 1)
        self.ID = ID
        self.daq9211 = daq9211
        self.lbl = tk.Label(self, text = 'Thermocouple: Channel %d' % self.ID, font = ("tkDefaultFont",14))
        self.lbl.grid(row = 0, column = 0, sticky = 'nw')

        #thermocouple type menu
        self.type_lbl = tk.Label(self, text = 'Thermocouple Type', font = ("tkDefaultFont",12))
        self.type_lbl.grid(row = 1, column = 0, columnspan = 2, sticky = 'nw' )
        self.type_str = tk.StringVar()
        #self.type_list = ["J","K","N","R","S","T","B","E"]
        self.type_list = ["K", "E"]
        self.type_str.set(self.type_list[0])
        self.type_menu = ttk.OptionMenu(self, self.type_str, self.type_list[0], *self.type_list)
        self.type_menu.grid(row = 2, column = 0, columnspan = 2, sticky = 'nw')

        self.min_lbl = tk.Label(self, text = 'Temperature Min', font = ("tkDefaultFont", 12))
        self.min_lbl.grid(row = 3, column = 0, sticky = 'nw')
        self.min_str = tk.StringVar()
        self.min_str.set('77')
        self.min_entry = tk.Entry(self, textvariable = self.min_str)
        self.min_entry.grid(row = 4, column = 0, sticky = 'nw')

        self.max_lbl = tk.Label(self, text = 'Temperature Max', font = ("tkDefaultFont", 12))
        self.max_lbl.grid(row = 3, column = 1, sticky = 'nw')
        self.max_str = tk.StringVar()
        self.max_str.set('1000')
        self.max_entry = tk.Entry(self, textvariable = self.max_str)
        self.max_entry.grid(row = 4, column = 1, padx = 10, sticky = 'nw')

        self.cjc_lbl = tk.Label(self, text = 'Cold Junction Compensation', font = ("tkDefaultFont",12))
        self.cjc_lbl.grid(row = 5, column = 0, sticky = 'nw' )
        self.cjc_str = tk.StringVar()
        self.cjc_list = ["Built-in", "Constant"]
        self.cjc_str.set(self.type_list[0])
        self.cjc_menu = ttk.OptionMenu(self, self.cjc_str, self.cjc_list[0], *self.cjc_list)
        self.cjc_menu.grid(row = 6, column = 0, columnspan = 2, sticky = 'nw')

        self.config_btn = ttk.Button(self, text = 'Configure Channel', command = lambda: self.config_channel())
        self.config_btn.grid(row = 0, column = 2, rowspan = 8, padx = 5, sticky = 'nsew')

    def config_channel(self):
        #master is the mainchannel frame
        #master.master is daq_config_frame
        #self.ID is the channel ID
        #add channel information to listbox in master.master.config_listbox
        tc = thermocouple(self.type_str.get(), self.min_str.get(), self.max_str.get(), self.cjc_str.get())
        chan = channel('%s/ai%d' % (self.daq9211.name, self.ID), tc, int(self.ID))
        self.daq9211.add_channel(chan)
        item_str = "Channel: %s" % (chan.name)
        time_item_str = '[%s] %s' % (str(datetime.datetime.now().time().strftime("%H:%M:%S")),item_str)
        #first get all elements in the listbox
        all_elements = self.master.master.config_listbox.get(0,tk.END)
        for index, elem in enumerate(all_elements):
            if item_str in elem: #then the channel corresponding to index has been configured and needs to be overwritten
                self.master.master.config_listbox.delete(index)
        self.master.master.config_listbox.insert(tk.END, time_item_str)

class daq_voltage_frame(tk.Frame):
    def __init__(self, master, controller, daq9211, ID):
        tk.Frame.__init__(self, master)
        self.config(borderwidth = 5, relief = tk.GROOVE, width = 100, height = 50)
        self.grid_rowconfigure(1,weight = 1)
        self.grid_columnconfigure(0, weight = 1)
        self.grid_columnconfigure(1, weight = 1)
       # self.grid_columnconfigure(2, weight = 1)
        self.ID = ID
        self.daq9211 = daq9211
        self.lbl = tk.Label(self, text = 'Voltage: Channel %d' % self.ID, font = ("tkDefaultFont",14))
        self.lbl.grid(row = 0, column = 0, sticky = 'nw')
        self.config_btn = ttk.Button(self, text = 'Configure Channel', command = lambda: self.config_channel())
        self.config_btn.grid(row = 0, column = 2, rowspan = 2, padx = 5, sticky = 'nsew')

    def config_channel(self):
        #master is the mainchannel frame
        #master.master is daq_config_frame
        #self.ID is the channel ID
        #add channel information to listbox in master.master.config_listbox
        tc = voltage(self.ID)
        chan = channel('%s/ai%d' % (self.daq9211.name, self.ID), tc, int(self.ID))
        self.daq9211.add_channel(chan)
        item_str = "Channel: %s" % (chan.name)
        time_item_str = '[%s] %s' % (str(datetime.datetime.now().time().strftime("%H:%M:%S")),item_str)
        #first get all elements in the listbox
        all_elements = self.master.master.config_listbox.get(0,tk.END)
        for index, elem in enumerate(all_elements):
            if item_str in elem: #then the channel corresponding to index has been configured and needs to be overwritten
                self.master.master.config_listbox.delete(index)
        self.master.master.config_listbox.insert(tk.END, time_item_str)


import os

import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid import AxesGrid

from skimage.io import MultiImage
from skimage import data_dir

# Load the multi-layer image
fname = os.path.join(data_dir, 'multipage.tif')
img = MultiImage(fname)

# Create an image grid
fig = plt.figure()
grid = AxesGrid(fig,
                rect=(1, 1, 1),
                nrows_ncols=(1, 2),
                axes_pad=0.1)

# Plot the layers on the image grid
for i, frame in enumerate(img):
    grid[i].imshow(frame, cmap=plt.cm.gray)
    grid[i].set_xlabel('Frame %s' % i)
    grid[i].set_xticks([])
    grid[i].set_yticks([])

plt.show()

Dash Docsets/scikit-image 0.9.x 2.docset/Contents/Resources/Documents/plots/show_collection.py


import argparse
import numpy as np
from pulp import *
import random
import math

parser = argparse.ArgumentParser()

parser.add_argument('--mdp', required = True)
parser.add_argument('--algorithm')
parser.add_argument('--policy')

args = parser.parse_args()

def storeMDP(mdpPath):
    with open(mdpPath) as file:
        rows = file.readlines()

    num_S = int(rows[0].split()[1])
    num_A = int(rows[1].split()[1])
    end = [int(ending) for ending in rows[2].split()[1:]] # end is an array

    T = np.zeros((num_S, num_A, num_S))
    R = np.zeros((num_S, num_A, num_S))

    discount = float(rows[-1].split()[1])
    typ = rows[-2].split()[1]

    for row in rows[3:]:

        if row.split()[0] == 'transition':
            row = row.split()
            s = int(row[1])
            a = int(row[2])
            s_ = int(row[3])
            rew = float(row[4])
            p = float(row[5])

            T[s, a, s_] = p
            R[s, a, s_] = rew

    return num_S, num_A, T, R, end, typ, discount

def valueIteration(S, A, T, R, discount):

    V = np.random.uniform(0, 1, S)
    V_ = np.random.uniform(0, 1, S)*0.5
    pi = np.zeros(S)

    while np.linalg.norm(V-V_) > 1e-9:
        V_ = V.copy()

        for s in range(S):
            max_val = 0
            actions = []
            for a in range(A):
                curr_sum = 0
                for s_ in range(S):
                    p = T[s, a, s_]

                    curr_sum += p*(R[s, a, s_] + discount * V[s_])
                actions.append(curr_sum)
                max_val = max(max_val, curr_sum)
            V[s] = max_val
            pi[s] = int(np.argmax(actions))

    ppi = pi.astype(int)
    return V, ppi

def LinearProgramming(S, A, T, R, discount):

    problem = LpProblem('MDP', LpMinimize)
    V = LpVariable.dicts('V',list(range(S)))

    problem += lpSum([V[s] for s in range(S)])
    pi = np.zeros(S)

    for s in range(S):
        for a in range(A):
            exp = 0
            for s_ in range(S):
                if T[s, a, s_] != 0:
                    exp += T[s, a, s_]*(R[s, a, s_] + discount*V[s_])
            problem += lpSum([exp])<=V[s]

    problem.solve(PULP_CBC_CMD(msg=0))
    for s in range(S):
        V[s] = pulp.value(V[s])
    V = np.array(list(V.values()), dtype=float)

    for s in range(S):
        actions = []
        for a in range(A):
            exp = 0
            for s_ in range(S):
                exp += T[s, a, s_]*(R[s, a, s_]  + discount*V[s_])
            actions.append(exp)
        pi[s] = np.argmax(actions)

    ppi = pi.astype(int)

    return V, ppi

def PolicyIteration(S, A, T, R, discount):

    V = np.zeros(S)
    pi = np.zeros(S)

    prev_policy = np.zeros(S)
    policy = [0 for s in range(S)]

    change = True

    while change:
        count = 0
        change = False

        for s in range(S):
            V[s] = sum([T[s,policy[s],s_] * (R[s,policy[s],s_] + discount*V[s_]) for s_ in range(S)])

        for s in range(S):
            best = V[s]
            for a in range(A):
                what = sum([T[s, a, s_] * (R[s, a, s_] + discount*V[s_]) for s_ in range(S)])
                if what > best:
                    count += 1
                    change = True
                    policy[s] = a
                    best = what
    return V, policy

def calcPolicy(policyPath, S):

    policy = []

    with open(policyPath) as file:
        rows = file.readlines()

    for row in rows:
        policy.append(int(row))

    return policy

def PolicyEvaluation(S, A, T, R, discount, policy):

    policy = calcPolicy(policy, S)
    V = np.zeros(S)

    while True:
        delta = 0
        for s in range(S):
            v = 0
            a = policy[s]
            for s_ in range(S):

                v += T[s, a, s_] * (R[s, a, s_] + discount * V[s_])
            delta = max(delta, np.abs(v-V[s]))
            V[s] = v

        if delta < 1e-8:
            break

    return V, policy
if __name__ == '__main__':
    mdpPath = args.mdp

    S, A, T, R, end, mdp_type, discount = storeMDP(mdpPath)
    #print(valueIteration(S, A, T, R, discount))

    if  args.policy:
        polPath = args.policy
        V, pi = PolicyEvaluation(S, A, T, R, discount, polPath)

    elif args.algorithm:

        if args.algorithm == 'vi':
            V, pi = valueIteration(S, A, T, R, discount)
        elif args.algorithm == 'lp':
            V, pi = LinearProgramming(S, A, T, R, discount)
        elif args.algorithm == 'hpi':
            V, pi = PolicyIteration(S, A, T, R, discount)
    else:
        V, pi = LinearProgramming(S, A, T, R, discount)

    for s in range(S):
        print("{0:6f}".format(V[s]), pi[s])


import evaluate
import torch
import torch.nn.functional as F

from accelerate import Accelerator
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from tqdm.auto import tqdm
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    get_scheduler
)

raw_datasets = load_dataset("glue", "mrpc")
checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

# tokenize dataset + prepare it so that it is acceptable to PyTorch Dataloader object
def tokenize_function(example):
    return tokenizer(example["sentence1"], example["sentence2"], truncation=True)

tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

tokenized_datasets = tokenized_datasets.remove_columns(["sentence1", "sentence2", "idx"])
tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
tokenized_datasets.set_format("torch")
tokenized_datasets["train"].column_names # Check that this is columns model will accept

# Create dataloader objects
train_dataloader = DataLoader(
    tokenized_datasets["train"], shuffle=True, batch_size=8, collate_fn=data_collator
)
eval_dataloader = DataLoader(
    tokenized_datasets["validation"], batch_size=8, collate_fn=data_collator
)

# Create Accelerator
accelerator = Accelerator()

# Create model
model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)

# Optimizer
optimizer = AdamW(model.parameters(), lr=5e-5)

# Learning Rate Scheduler
num_epochs = 3
num_training_steps = num_epochs * len(train_dataloader)
lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps,
)

# Configure to GPU if GPU is available
train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(
    train_dataloader, eval_dataloader, model, optimizer
)

# Training loop
progress_bar = tqdm(range(num_training_steps))

model.train()
for epoch in range(num_epochs):
    for batch in train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

# Evaluation Loop
metric = evaluate.load("glue", "mrpc")
model.eval()
for batch in eval_dataloader:
    with torch.no_grad():
        outputs = model(**batch)

    logits = outputs.logits
    predictions = torch.argmax(logits, dim=-1)
    metric.add_batch(predictions=predictions, references=batch["labels"])

metric.compute()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 10 10:39:27 2021

@author: mac
"""

# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

import numpy as np
from sympy import sieve as euclid

## source: https://stackoverflow.com/questions/2068372/fastest-way-to-list-all-primes-below-n
def primesfrom2to(n):
    """ Input n>=6, Returns a array of primes, 2 <= p < n """
    sieve = np.ones(n//3 + (n%6==2), dtype=np.bool)
    for i in range(1,int(n**0.5)//3+1):
        if sieve[i]:
            k=3*i+1|1
            sieve[       k*k//3     ::2*k] = False
            sieve[k*(k-2*(i&1)+4)//3::2*k] = False
    return np.r_[2,3,((3*np.nonzero(sieve)[0][1:]+1)|1)]

## or we could use sympy:
def erastothenes(n):

    euclid._reset()

    euclid.extend(n)

    return np.array(euclid._list)

## create csv file of prime encodings for all primes less than 10**5:
N = 10**9

X = np.arange(1,N+1)

X_ = primesfrom2to(N)

def prime_or_not(X,X_):

    """A boolean map that decides whether an integer is prime or not."""

    N = max(X_)

    Y = np.zeros(len(X))

    j = 0

    for i in range(N):

        if X[i] == X_[j]:
            Y[i] = 1
            j += 1

        else:
            Y[i] = 0

    return Y

def binary_encoding(X,  num_bits):

    encoder = "{:0" + str(num_bits) + "b}"

    binaries = np.zeros((len(X),num_bits))

    for i in range(len(X)):

        num = encoder.format(X[i])

        binaries[i] = [int(x) for x in str(num)]

    return binaries

### create dataframe:
import pandas as pd

dataset_X = pd.DataFrame(binary_encoding(X, 32))

dataset_Y = pd.DataFrame({'Y': prime_or_not(X,X_)})

data = pd.concat([dataset_X,dataset_Y],axis = 1)

path = 'your_path'

data.to_csv(path,index=False)


import streamlit as st
from datetime import date
import yfinance as yf
from prophet import Prophet
import plotly.graph_objects as go

st.title('Stock Forecast App')

stocks = ('GOOG', 'AAPL', 'MSFT', 'AMZN','META','TSLA','BRK.A','JNJ','KO','LLY')
selected_stock = st.selectbox('Select dataset for prediction', stocks)

# User defined parameters
years = st.slider('Years of prediction:', 1, 10)
period = years * 365 # days

# Load data
start_date = '2010-01-01'
today = date.today().strftime("%Y-%m-%d")

data = yf.download(selected_stock, start_date, today)

# Preview data
st.subheader('Raw data')
st.write(data.tail())

# Train Prophet model
df_train = data[['Close']]
df_train = df_train.reset_index()
df_train = df_train.rename(columns={"Date": "ds", "Close": "y"})

model = Prophet()
model.fit(df_train)

# Make prediction
future = model.make_future_dataframe(periods=period)
forecast = model.predict(future)

# Show and plot forecast
st.subheader('Forecast data')
st.write(forecast.tail())

st.write(f'Forecast plot for {years} years')
fig1 = model.plot(forecast)
st.write(fig1)

st.write(f'Forecast components')
fig2 = model.plot_components(forecast)
st.write(fig2)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Increase version and commit changes to git """

from __future__ import print_function

import subprocess
import re
import sys

versionFile = "src/dist.py"
readmeFile = "README.md"

def check_readme(version):
	version = '.'.join(map(str, version))
	changelog = []
	copying = False
	r = re.compile(r'^## %s\s+$' % version)
	with open(readmeFile) as f:
		lines = f.readlines()
		for line in lines:
			if r.match(line):
				copying = True
			elif line.startswith('#'):
				copying = False
			elif copying:
				line = line.rstrip()
				if line:
					changelog.append(line)
	if not changelog:
		raise Exception("Please provide release notes for version %s" % version)

	changelog = "\n".join(["[size=18pt]Версия %s[/size]" % version] + changelog)
	print("\nChangelog bb:")
	print(changelog)
	return changelog

def clip(s):
	subprocess.Popen(['clip'], stdin=subprocess.PIPE).communicate(s)
	print("Copied to clipboard")

def main(skip_push):
	subprocess.check_call(['git', 'reset'])
	c = subprocess.call(['git', 'diff', '--exit-code', '--', versionFile, readmeFile])
	if c != 0:
		print("Aborted because %s or %s has modifications!" % (versionFile, readmeFile))
		sys.exit(1)
	else:
		nextVer = None
		with open(versionFile, 'r') as f:
			lines = f.readlines()
			r = re.compile(r'^\s*VERSION\s*=\s*"(.*)"$')
			for i, line in enumerate(lines):
				m = r.match(line)
				if m:
					ver = m.group(1)
					print("Current version", ver)
					ver = list(map(int, ver.split('.')))
					nextVer = ver[:2]
					nextVer[1] += 1
					lines[i] = 'VERSION = "%s"' % '.'.join(map(str, nextVer))
					print("Update to", lines[i])

		log = check_readme(nextVer)

		with open(versionFile, 'wb') as f:
			f.writelines(lines)
			f.write('\n')

		assert nextVer is not None

		subprocess.check_call(['git', 'add', versionFile])
		ver = '.'.join(map(str, nextVer))
		subprocess.check_call(['git', 'commit', '-m', 'version up %s' % ver])
		subprocess.check_call(['git', 'tag', '-a', 'v/%s' % ver, '-m', 'version up %s' % ver])
		if not skip_push:
			subprocess.check_call(['git', 'push'])

	print("Done.\n")

	if sys.platform == 'win32':
		clip(log)
		import json
		with open('secret.json') as f:
			url = json.load(f)['forum']
		subprocess.call(["C:/Program Files/Mozilla Firefox/firefox.exe", url])

if __name__ == "__main__":
	main(len(sys.argv) > 1 and '-n' in sys.argv[1:])


from django.shortcuts import render
from .forms import ContactForm, SignUpForm
def home(request):
	title = "Welcome"
	form = SignUpForm(request.POST or None)

	context = {
		"title": title,
		"form": form
		}

	if form.is_valid():

		instance = form.save(commit = False)
		if not instance.full_name:
			instance.full_name = "Hardik"
		instance.save()
		context = {
		"title": "Thank you"
		}

	return render(request,"home.html", context)

def contact(request):
	title = 'Contact Us'
	form = ContactForm(request.POST or None)

	context ={
		"form": form,
		"title":title,
	}
	return render(request,"forms.html", context)


import numpy as np
import scipy.io as sio
from datetime import datetime, timedelta
import pandas as pd
import os
import csv
import matplotlib.pyplot as plt
from matplotlib.ticker import ScalarFormatter, MultipleLocator
import matplotlib.mlab as mlab
import scipy as sp
from scipy.interpolate import UnivariateSpline
import scipy.interpolate as si
from scipy.interpolate import interp1d

base_dir = os.path.expanduser('~')
path_data=base_dir+'/Dropbox/Monash_Uni/SO/MAC/Data/YOTC/mat/'

#*****************************************************************************\
#Default Info
mac = {'name': 'Macquarie Island, Australia', 'lat': -54.62, 'lon': 158.85}
lat_mac = mac['lat']
lon_mac = mac['lon']

ptemp_thold_main=0.010           # K/m
ptemp_thold_sec=0.005            # K/m
shear_thold=0.015               # 1/s

#*****************************************************************************\
#*****************************************************************************\
#                            YOTC Data
#*****************************************************************************\
# #*****************************************************************************\
mat= sio.loadmat(path_data+'yotc_data.mat')
#*****************************************************************************\
#Crear fecha de inicio leyendo time
time= mat['time'][0]
date_ini=datetime(1900, 1, 1) + timedelta(hours=int(time[0])) #hours since
#Arreglo de fechas
date_yotc = pd.date_range(date_ini, periods=len(time), freq='12H')
#*****************************************************************************\
#Reading variables
t_ori= mat['temp'][:] #K
u_ori= mat['u'][:]
v_ori= mat['v'][:]
q_ori= mat['q'][:]

#rotate from surface to free atmosphere
temp=t_ori[:,::-1]
u=u_ori[:,::-1]
v=v_ori[:,::-1]
q=q_ori[:,::-1] #kg/kg
mixr= q*1000 #g/kg

#*****************************************************************************\
#Leyendo Alturas y press
file_levels = np.genfromtxt('./Read_Files/YOTC/levels.csv', delimiter=',')
hlev_yotc=file_levels[:,6]
#plev_yotc=file_levels[:,3]
plev_yotc=file_levels[:,4] #value 10 is 925

#*****************************************************************************\
g=9.8 #m seg^-2

#Calculate Virtual Temperature
theta=(temp)*(1000./plev_yotc)**0.287;
#Calculate Potential Temperature
pot_temp=(1+0.61*(mixr/1000.))*theta;

pot_temp_grad=np.zeros(pot_temp.shape)
yvert_shear=np.empty(pot_temp.shape)*np.nan
brn_yotca=np.empty(pot_temp.shape)*np.nan
#Calculate Wind Shear and Gradiente Potential Temp
for j in range(0,len(time)):
    for i in range(1,len(hlev_yotc)-1):
        pot_temp_grad[j,i]=(pot_temp[j,i+1]-pot_temp[j,i])/float(hlev_yotc[i+1]-hlev_yotc[i])
        yvert_shear[j,i]=np.sqrt(float((u[j,i]-u[j,i-1])**2+(v[j,i]-v[j,i-1])**2))/float(hlev_yotc[i]-hlev_yotc[i-1])

for j in range(0,len(time)):
    for i in range(1,len(hlev_yotc)):
        brn_yotca[j,i]=(g/float(pot_temp[j,0]))*((hlev_yotc[i]-hlev_yotc[0])*(pot_temp[j,i]-pot_temp[j,0]))/float((u[j,i]-u[j,0])**2+(v[j,i]-v[j,0])**2)

#Calculate Relative Humidity
relhum=0.263*plev_yotc*q*np.exp((17.67*(temp-273.16))/(temp-29.65))**(-1)*100
relhum[relhum>100]=100
relhum[relhum<0]=0

#******************************************************************************
#Boundary Layer Height Inversion 1 and 2
#Variables Initialization
sec_ind=np.empty(len(time))
#main_inv=np.empty(len(time),'float')
sec_inv=np.empty(len(time))
main_inversion=np.empty(len(time))
sec_inversion=np.empty(len(time))
main_inv_hght=np.empty(len(time))
sec_inv_hght=np.empty(len(time))
yotc_clas=np.empty(len(time))
yotc_depth=np.empty(len(time))
yotc_hght_1invBL=np.empty(len(time))
yotc_hght_2invBL=np.empty(len(time))
yotc_hght_1invDL=np.empty(len(time))
yotc_hght_2invDL=np.empty(len(time))
yotc_hght_1inv=np.empty(len(time))
yotc_hght_2inv=np.empty(len(time))
yotc_strg_1inv=np.empty(len(time))
yotc_strg_2inv=np.empty(len(time))

relhum_yotc=np.empty(len(time))
temp_yotc=np.empty(len(time))
u_yotc=np.empty(len(time))
v_yotc=np.empty(len(time))
mixr_yotc=np.empty(len(time))
pot_temp_yotc=np.empty(len(time))
brn_yotc=np.empty(len(time))

#******************************************************************************
#Main Inversion Position
for ind,line in enumerate(hlev_yotc):
    if line>=float(100.):
        twenty_y_index=ind
        break
for ind,line in enumerate(hlev_yotc):
    if line>=2500:
        twoky=ind
        break

main_inv=pot_temp_grad[:,twenty_y_index:twoky].argmax(axis=1)
[i for i, j in enumerate(pot_temp_grad[:,twenty_y_index:twoky]) if j == main_inv]
main_inv+=twenty_y_index #posicion main inv mas indice de sobre 100 m (3)

# Second Inversion Position
for i in range(0,len(time)):
    for ind in range(twenty_y_index,main_inv[i]):
    #height 2da inv 80% main inv
        if hlev_yotc[ind]>=(0.8)*hlev_yotc[main_inv[i]]:
            sec_ind[i]=ind
            break
        else:
            sec_ind[i]=np.nan
    if main_inv[i]==twenty_y_index:
        sec_ind[i]=np.nan
    #calcula la posicion de la sec inv (trata si se puede, si no asigna nan)
    try:
        sec_inv[i]=pot_temp_grad[i,twenty_y_index:sec_ind[i]].argmax(0)
        [z for z, j in enumerate(pot_temp_grad[i,twenty_y_index:sec_ind[i]]) if j == sec_inv[i]]
        sec_inv[i]+=twenty_y_index
    except:
        sec_inv[i]=np.nan

# main inversion must be > theta_v threshold
ptemp_comp1=pot_temp_grad[:,main_inv[:]].diagonal() #extrae diagonal de pot temp
for i in range(0,len(time)):
    if ptemp_comp1[i]<ptemp_thold_main:
        #main_inv[i]=np.nan
        main_inv[i]=-9999 # Cannot convert float NaN to integer
        main_inversion[i]=False
        sec_inv[i]=np.nan
    else:
        main_inv_hght[i]=hlev_yotc[main_inv[i]]
        main_inversion[i]=True

    if main_inv_hght[i]<=1:
        main_inv_hght[i]=np.nan #Corrige el -9999 para calcular alt

    # secondary inversion must be > theta_v threshold

    if np.isnan(sec_inv[i])==False and pot_temp_grad[i,sec_inv[i]]>=ptemp_thold_sec:
        sec_inversion[i]=True
        sec_inv_hght[i]=hlev_yotc[sec_inv[i]]
    else:
        sec_inversion[i]=False
        sec_inv_hght[i]=np.nan

    #Clasification
    if sec_inversion[i]==False and main_inversion[i]==True:
        yotc_clas[i]=2
        yotc_depth[i]=np.nan
        yotc_hght_1invBL[i]=np.nan
        yotc_hght_2invBL[i]=np.nan
        yotc_hght_1invDL[i]=np.nan
        yotc_hght_2invDL[i]=np.nan
        yotc_hght_1inv[i]=hlev_yotc[main_inv[i]]
        yotc_hght_2inv[i]=np.nan
        yotc_strg_1inv[i]=pot_temp_grad[i,main_inv[i]]
        yotc_strg_2inv[i]=np.nan
    elif sec_inversion[i]==False and main_inversion[i]==False:
        yotc_clas[i]=1
        yotc_depth[i]=np.nan
        yotc_hght_1invBL[i]=np.nan
        yotc_hght_2invBL[i]=np.nan
        yotc_hght_1invDL[i]=np.nan
        yotc_hght_2invDL[i]=np.nan
        yotc_hght_1inv[i]=np.nan
        yotc_hght_2inv[i]=np.nan
        yotc_strg_1inv[i]=np.nan
        yotc_strg_2inv[i]=np.nan
    elif main_inversion[i]==True and sec_inversion[i]==True and yvert_shear[i,sec_inv[i]]>=shear_thold:
        yotc_clas[i]=4
        yotc_depth[i]=(hlev_yotc[main_inv[i]]-hlev_yotc[sec_inv[i]])
        yotc_hght_1invBL[i]=hlev_yotc[main_inv[i]]
        yotc_hght_2invBL[i]=hlev_yotc[sec_inv[i]]
        yotc_hght_1invDL[i]=np.nan
        yotc_hght_2invDL[i]=np.nan
        yotc_hght_1inv[i]=hlev_yotc[main_inv[i]]
        yotc_hght_2inv[i]=hlev_yotc[sec_inv[i]]
        yotc_strg_1inv[i]=pot_temp_grad[i,main_inv[i]]
        yotc_strg_2inv[i]=pot_temp_grad[i,sec_inv[i]]
    else:
        yotc_clas[i]=3
        yotc_hght_1invDL[i]=hlev_yotc[main_inv[i]]
        yotc_hght_2invDL[i]=hlev_yotc[sec_inv[i]]
        yotc_depth[i]=(hlev_yotc[main_inv[i]]-hlev_yotc[sec_inv[i]])
        yotc_hght_1invBL[i]=np.nan
        yotc_hght_2invBL[i]=np.nan
        yotc_hght_1inv[i]=hlev_yotc[main_inv[i]]
        yotc_hght_2inv[i]=hlev_yotc[sec_inv[i]]
        yotc_strg_1inv[i]=pot_temp_grad[i,main_inv[i]]
        yotc_strg_2inv[i]=pot_temp_grad[i,sec_inv[i]]

    #print

    relhum_yotc[i]=relhum[i,10]
    temp_yotc[i]=temp[i,10]
    u_yotc[i]=u[i,10]
    v_yotc[i]=v[i,10]
    mixr_yotc[i]=mixr[i,10]
    pot_temp_yotc[i]=pot_temp[i,10]
    brn_yotc[i]=brn_yotca[i,10]
#np.count_nonzero(~np.isnan(main_inv))

#*****************************************************************************\
# # #Histogram
# n, bin_edges =np.histogram(yotc_clas, bins=[1, 2, 3, 4,5],normed=1)

# plt.figure(0)
# y=bin_edges[0:4]

# #clas = ('No Inv.', 'Single Inv.', 'Decoupled L.', 'Buffer L.')
# clas = ('NI', 'SI', 'DL', 'BL')
# y_pos = np.arange(len(clas))

# plt.barh(y_pos, n, align='center', color='green')
# plt.yticks(y_pos, clas)
# plt.xticks(np.arange(0, 1.1, 0.1))
# plt.xlabel('Performance')
# plt.xlim([0,1])
# #plt.show()

# ****************************************************************************\
# ****************************************************************************\
#                            MAC Data Original Levels
#*****************************************************************************\
# ****************************************************************************\
path_databom=base_dir+'/Dropbox/Monash_Uni/SO/MAC/MatFiles/files_bom/'
matb= sio.loadmat(path_databom+'BOM_2006-2010.mat')
#*****************************************************************************\

bom=matb['SD'][:]#(3000,8,3594)
ni=bom.shape
timesd= matb['timesd'][:]
#*****************************************************************************\
#Delete special cases
indexdel1=[]
indexdel2=[]
#Delete when array is only NaN
for j in range(0,ni[2]):
    if np.isnan(np.nansum(bom[:,0,j]))==True:
        indexdel1=np.append(indexdel1,j)

bom=np.delete(bom, indexdel1,axis=2)
ni=bom.shape
timesd=np.delete(timesd, indexdel1)

#Delete when max height is lower than 2500 mts
for j in range(0,ni[2]):
    if np.nanmax(bom[:,1,j])<2500:
        indexdel2=np.append(indexdel2,j)

bom=np.delete(bom, indexdel2,axis=2) #(3000,8,3545)
ni=bom.shape
timesd=np.delete(timesd, indexdel2)
#*****************************************************************************\

#Convert Matlab datenum into Python datetime source: https://gist.github.com/vicow)
def datenum_to_datetime(datenum):
    days = datenum % 1
    hours = days % 1 * 24
    minutes = hours % 1 * 60
    seconds = minutes % 1 * 60
    return datetime.fromordinal(int(datenum)) \
        + timedelta(days=int(days)) \
        + timedelta(hours=int(hours)) \
        + timedelta(minutes=int(minutes)) \
        + timedelta(seconds=round(seconds)) \
        - timedelta(days=366)

#*****************************************************************************\
#Separation of Variables
pres=bom[:,0,:].reshape(ni[0],ni[2])
hght=bom[:,1,:].reshape(ni[0],ni[2])
temp=bom[:,2,:].reshape(ni[0],ni[2])
mixr=bom[:,5,:].reshape(ni[0],ni[2])
wdir_initial=bom[:,6,:].reshape(ni[0],ni[2])
wspd=bom[:,7,:].reshape(ni[0],ni[2])
relh=bom[:,4,:].reshape(ni[0],ni[2])

# vni=-wspd*(np.cos(np.radians(wdir_initial)))
# uni=-wspd*(np.sin(np.radians(wdir_initial)))

u=wspd*(np.cos(np.radians(270-wdir_initial)))
v=wspd*(np.sin(np.radians(270-wdir_initial)))

#*****************************************************************************\
#*****************************************************************************\
#                            MAC Data YOTC Levels
#*****************************************************************************\
#*****************************************************************************\
#Interpolation to YOTC Levels
# plt.plot(hght[0:200,1], 'ro')
# plt.show()

prutemp=np.empty((len(hlev_yotc),0))
prumixr=np.empty((len(hlev_yotc),0))
# pruwspd=np.empty((len(hlev_yotc),0))
# pruwdir=np.empty((len(hlev_yotc),0))
pruu=np.empty((len(hlev_yotc),0))
pruv=np.empty((len(hlev_yotc),0))
prurelh=np.empty((len(hlev_yotc),0))

for j in range(0,ni[2]):
#for j in range(0,100):
#height initialization
    x=hght[:,j]
    x[-1]=np.nan
    new_x=hlev_yotc
#Interpolation YOTC levels
    yt=temp[:,j]
    rest=interp1d(x,yt)(new_x)
    prutemp=np.append(prutemp,rest)

    ym=mixr[:,j]
    resm=interp1d(x,ym)(new_x)
    prumixr=np.append(prumixr,resm)

    yw=u[:,j]
    resw=interp1d(x,yw)(new_x)
    pruu=np.append(pruu,resw)

    yd=v[:,j]
    resd=interp1d(x,yd)(new_x)
    pruv=np.append(pruv,resd)

    yr=relh[:,j]
    resr=interp1d(x,yr)(new_x)
    prurelh=np.append(prurelh,resr)

tempmac_ylev=prutemp.reshape(-1,len(hlev_yotc)).transpose()
umac_ylev=pruu.reshape(-1,len(hlev_yotc)).transpose()
vmac_ylev=pruv.reshape(-1,len(hlev_yotc)).transpose()
mixrmac_ylev=prumixr.reshape(-1,len(hlev_yotc)).transpose()
relhmac_ylev=prurelh.reshape(-1,len(hlev_yotc)).transpose()

# wspdmac_ylev=pruwspd.reshape(-1,len(hlev_yotc)).transpose()
# wdirmac_ylev=pruwdir.reshape(-1,len(hlev_yotc)).transpose()

wspdmac_ylev=np.sqrt(umac_ylev**2 + vmac_ylev**2)
wdirmac_ylev=np.arctan2(-umac_ylev, -vmac_ylev)*(180/np.pi)
wdirmac_ylev[(umac_ylev == 0) & (vmac_ylev == 0)]=0

#*****************************************************************************\
#Initialization Variables
wdir_my=np.empty(tempmac_ylev.shape)

spec_hum_my=np.empty(tempmac_ylev.shape)
tempv_my=np.empty(tempmac_ylev.shape)
ptemp_my=np.empty(tempmac_ylev.shape)
brn_mya=np.empty(tempmac_ylev.shape)

ucomp_my=np.empty(tempmac_ylev.shape)
vcomp_my=np.empty(tempmac_ylev.shape)

ucomp_initial_my=np.empty(tempmac_ylev.shape)
vcomp_initial_my=np.empty(tempmac_ylev.shape)

vert_shear_my=np.empty(tempmac_ylev.shape)*np.nan
ptemp_gmy=np.empty(tempmac_ylev.shape)*np.nan
main_my_inv=np.empty(ni[2])*np.nan
twenty_my_index=[]
twokmy=[]
sec_my_ind=np.empty(ni[2])*np.nan
sec_my_inv=np.empty(ni[2])*np.nan
ptemp_comp2=np.empty(ni[2])*np.nan
main_my_inv_hght=np.empty(ni[2])*np.nan
main_my_inversion=np.empty(ni[2])*np.nan
sec_my_inv_hght=np.empty(ni[2])*np.nan
sec_my_inversion=np.empty(ni[2])*np.nan
mac_y_clas=np.empty(ni[2])*np.nan
mac_y_hght_1invDL=np.empty(ni[2])*np.nan
mac_y_hght_2invDL=np.empty(ni[2])*np.nan
mac_y_depth=np.empty(ni[2])*np.nan
mac_y_hght_1invBL=np.empty(ni[2])*np.nan
mac_y_hght_2invBL=np.empty(ni[2])*np.nan
mac_y_hght_1inv=np.empty(ni[2])*np.nan
mac_y_hght_2inv=np.empty(ni[2])*np.nan
mac_y_strg_1inv=np.empty(ni[2])*np.nan
mac_y_strg_2inv=np.empty(ni[2])*np.nan

relhum_my=np.empty(ni[2])*np.nan
temp_my=np.empty(ni[2])*np.nan
u_my=np.empty(ni[2])*np.nan
v_my=np.empty(ni[2])*np.nan
mixr_my=np.empty(ni[2])*np.nan
pot_temp_my=np.empty(ni[2])*np.nan
brn_my=np.empty(ni[2])*np.nan

#*****************************************************************************\
for j in range(0,ni[2]):
#for j in range(0,100):
#Calculate new variables
    for i in range(0,len(hlev_yotc)):
        if 0.<=wdirmac_ylev[i,j]<=90.:
            wdir_my[i,j]=wdirmac_ylev[i,j]+270.
        elif 90.<=wdirmac_ylev[i,j]<=360.:
            wdir_my[i,j]=wdirmac_ylev[i,j]-90.

        spec_hum_my[i,j]=(float(mixrmac_ylev[i,j])/1000.)/(1+(float(mixrmac_ylev[i,j])/1000.))

        tempv_my[i,j]=tempmac_ylev[i,j]*float(1+0.61*spec_hum_my[i,j])
        ptemp_my[i,j]=(tempv_my[i,j]+273.16)*((1000./plev_yotc[i])**0.286) #Ok

        brn_mya[i,j]=(g/float(ptemp_my[0,j]))*((hlev_yotc[i]-hlev_yotc[0])*(ptemp_my[i,j]-ptemp_my[0,j]))/float((umac_ylev[i,j]-umac_ylev[0,j])**2+(vmac_ylev[i,j]-vmac_ylev[0,j])**2)

        # vcomp_my[i,j]=-wspdmac_ylev[i,j]*(np.cos(np.radians(wdir_my[i,j])))
        # ucomp_my[i,j]=-wspdmac_ylev[i,j]*(np.sin(np.radians(wdir_my[i,j])))

        # vcomp_initial_my[i,j]=-wspdmac_ylev[i,j]*(np.cos(np.radians(wdirmac_ylev[i,j])))
        # ucomp_initial_my[i,j]=-wspdmac_ylev[i,j]*(np.sin(np.radians(wdirmac_ylev[i,j])))

    for i in range(0,len(hlev_yotc)-1):
        vert_shear_my[i,j]=np.sqrt(float((umac_ylev[i,j]-umac_ylev[i-1,j])**2+(vmac_ylev[i,j]-vmac_ylev[i-1,j])**2))/float(hlev_yotc[i+1]-hlev_yotc[i])

        ptemp_gmy[i,j]=(ptemp_my[i+1,j]-ptemp_my[i,j])/float(hlev_yotc[i+1]-hlev_yotc[i])

    vert_shear_my[-1,j]=np.nan
    ptemp_gmy[-1,j]=np.nan

#*****************************************************************************\
# #Main Inversion Position
for ind,line in enumerate(hlev_yotc):
    if line>=float(100.):
        twenty_my_index=ind
        break
for ind,line in enumerate(hlev_yotc):
    if line>=2500:
        twokmy=ind
        break

main_my_inv=ptemp_gmy[twenty_my_index:twokmy,:].argmax(axis=0)
main_my_inv+=twenty_my_index #posicion main inv mas indice de sobre 100 m (3)

# # Second Inversion Position
for j in range(0,ni[2]):
#for j in range(0,100):
    for ind in range(twenty_my_index,main_my_inv[j]):
    #height 2da inv 80% main inv
        if hlev_yotc[ind]>=(0.8)*hlev_yotc[main_my_inv[j]]:
            sec_my_ind[j]=ind
            break
        else:
            sec_my_ind[j]=np.nan
    if main_my_inv[j]==twenty_my_index:
        sec_my_ind[j]=np.nan
    #calcula la posicion de la sec inv (trata si se puede, si no asigna nan)
    try:
        sec_my_inv[j]=ptemp_gmy[twenty_my_index:sec_my_ind[j],j].argmax(0)
        sec_my_inv[j]+=twenty_my_index
    except:
        sec_my_inv[j]=np.nan

# main inversion must be > theta_v threshold
for j in range(0,ni[2]):
#for j in range(0,100):
    ptemp_comp2[j]=ptemp_gmy[main_my_inv[j],j]#.diagonal() #extrae diagonal de pot temp
    if ptemp_comp2[j]<ptemp_thold_main:
        #main_inv[i]=np.nan
        main_my_inv[j]=-9999 # Cannot convert float NaN to integer
        main_my_inversion[j]=False
        sec_my_inv[j]=np.nan
    else:
        main_my_inv_hght[j]=hlev_yotc[main_my_inv[j]]
        main_my_inversion[j]=True

    if main_my_inv_hght[j]<=1:
        main_my_inv_hght[j]=np.nan #Corrige el -9999 para calcular alt

# secondary inversion must be > theta_v threshold

    if np.isnan(sec_my_inv[j])==False and ptemp_gmy[sec_my_inv[j],j]>=ptemp_thold_sec:
        sec_my_inversion[j]=True
        sec_my_inv_hght[j]=hlev_yotc[sec_my_inv[j]]
    else:
        sec_my_inversion[j]=False
        sec_my_inv_hght[j]=np.nan

# #Clasification
    if sec_my_inversion[j]==False and main_my_inversion[j]==True:
        mac_y_clas[j]=2
        mac_y_depth[j]=np.nan
        mac_y_hght_1invBL[j]=np.nan
        mac_y_hght_2invBL[j]=np.nan
        mac_y_hght_1invDL[j]=np.nan
        mac_y_hght_2invDL[j]=np.nan
        mac_y_hght_1inv[j]=hlev_yotc[main_my_inv[j]]
        mac_y_hght_2inv[j]=np.nan
        mac_y_strg_1inv[j]=ptemp_gmy[main_my_inv[j],j]
        mac_y_strg_2inv[j]=np.nan
    elif sec_my_inversion[j]==False and main_my_inversion[j]==False:
        mac_y_clas[j]=1
        mac_y_depth[j]=np.nan
        mac_y_hght_1invBL[j]=np.nan
        mac_y_hght_2invBL[j]=np.nan
        mac_y_hght_1invDL[j]=np.nan
        mac_y_hght_2invDL[j]=np.nan
        mac_y_hght_1inv[j]=np.nan
        mac_y_hght_2inv[j]=np.nan
        mac_y_strg_1inv[j]=np.nan
        mac_y_strg_2inv[j]=np.nan
    elif main_my_inversion[j]==True and sec_my_inversion[j]==True and vert_shear_my[sec_my_inv[j],j]>=shear_thold:
        mac_y_clas[j]=4
        mac_y_depth[j]=(hlev_yotc[main_my_inv[j]]-hlev_yotc[sec_my_inv[j]])
        mac_y_hght_1invBL[j]=hlev_yotc[main_my_inv[j]]
        mac_y_hght_2invBL[j]=hlev_yotc[sec_my_inv[j]]
        mac_y_hght_1invDL[j]=np.nan
        mac_y_hght_2invDL[j]=np.nan
        mac_y_hght_1inv[j]=hlev_yotc[main_my_inv[j]]
        mac_y_hght_2inv[j]=hlev_yotc[sec_my_inv[j]]
        mac_y_strg_1inv[j]=ptemp_gmy[main_my_inv[j],j]
        mac_y_strg_2inv[j]=ptemp_gmy[sec_my_inv[j],j]
    else:
        mac_y_clas[j]=3
        mac_y_hght_1invDL[j]=hlev_yotc[main_my_inv[j]]
        mac_y_hght_2invDL[j]=hlev_yotc[sec_my_inv[j]]
        mac_y_depth[j]=(hlev_yotc[main_my_inv[j]]-hlev_yotc[sec_my_inv[j]])
        mac_y_hght_1invBL[j]=np.nan
        mac_y_hght_2invBL[j]=np.nan
        mac_y_hght_1inv[j]=hlev_yotc[main_my_inv[j]]
        mac_y_hght_2inv[j]=hlev_yotc[sec_my_inv[j]]
        mac_y_strg_1inv[j]=ptemp_gmy[main_my_inv[j],j]
        mac_y_strg_2inv[j]=ptemp_gmy[sec_my_inv[j],j]

    relhum_my[j]=relhmac_ylev[10,j]
    temp_my[j]=tempmac_ylev[10,j]+273.16
    u_my[j]=umac_ylev[10,j]
    v_my[j]=vmac_ylev[10,j]
    mixr_my[j]=mixrmac_ylev[10,j]
    pot_temp_my[j]=ptemp_my[10,j]
    brn_my[j]=brn_mya[10,j]
#*****************************************************************************\
# #Histogram
# n, bin_edges =np.histogram(mac_y_clas, bins=[1, 2, 3, 4,5],normed=1)

# plt.figure(0)
# y=bin_edges[0:4]

# #clas = ('No Inv.', 'Single Inv.', 'Decoupled L.', 'Buffer L.')
# clas = ('NI', 'SI', 'DL', 'BL')
# y_pos = np.arange(len(clas))

# plt.barh(y_pos, n, align='center', color='green')
# plt.yticks(y_pos, clas)
# plt.xticks(np.arange(0, 1.1, 0.1))
# plt.xlabel('Performance')
# plt.xlim([0,1])
# #plt.show()

#*****************************************************************************\
#Cambiar fechas
timestamp = [datenum_to_datetime(t) for t in timesd]
time_my = np.array(timestamp)
time_my_ori = np.array(timestamp)

for i in range(0,ni[2]):
    #Cuando cae 23 horas del 31 de diciembre agrega un anio
    if time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==12:
        y1=time_my[i].year
        time_my[i]=time_my[i].replace(year=y1+1,hour=0, month=1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==1:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==3:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==5:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==7:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==8:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==31 and time_my[i].month==10:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==30 and time_my[i].month==4:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==30 and time_my[i].month==6:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==30 and time_my[i].month==9:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==30 and time_my[i].month==11:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    if  time_my[i].hour==23 and time_my[i].day==28 and time_my[i].month==2:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)
    #Bisiesto 2008
    if  time_my[i].hour==23 and time_my[i].day==29 and time_my[i].month==4 and time_my[i].year==2008:
        m1=time_my[i].month
        time_my[i]=time_my[i].replace(hour=0, month=m1+1,day=1)

    #Cuando cae 23 horas, mueve hora a las 00 del dia siguiente
    if time_my[i].hour==23:
        d1=time_my[i].day
        time_my[i]=time_my[i].replace(hour=0,day=d1+1)
    else:
        time_my[i]=time_my[i]
    #Cuando cae 11 horas, mueve hora a las 12 del mismo dia
    if time_my[i].hour==11:
        time_my[i]=time_my[i].replace(hour=12)
    else:
        time_my[i]=time_my[i]

    #Cuando cae 1 horas, mueve hora a las 0 del mismo dia
    if time_my[i].hour==1:
        time_my[i]=time_my[i].replace(hour=0)
    else:
        time_my[i]=time_my[i]

#*****************************************************************************\
#*****************************************************************************\
#                          Dataframes 2006-2010                               \
#*****************************************************************************\
#*****************************************************************************\
#Date index del periodo 2006-2010
date_index_all = pd.date_range('2006-01-01 00:00', periods=3652, freq='12H')
# #*****************************************************************************\
#Dataframe YOTC 2008-2010
dy={'Clas':yotc_clas,
'Depth':yotc_depth,
#'1 Inv BL': yotc_hght_1invBL,
#'2 Inv BL': yotc_hght_2invBL,
#'1 Inv DL': yotc_hght_1invDL,
#'2 Inv DL': yotc_hght_2invDL,
'1ra Inv': yotc_hght_1inv,
'2da Inv': yotc_hght_2inv,
'Strg 1inv': yotc_strg_1inv,
'Strg 2inv': yotc_strg_2inv,
'Mix R 925':mixr_yotc,
'T 925':temp_yotc,
'Theta v 925':pot_temp_yotc,
'u 925':u_yotc,
'v 925':v_yotc,
'RH 925':relhum_yotc}
#'BRN 925':brn_yotc}

df_yotc = pd.DataFrame(data=dy,index=date_yotc)
df_yotc.index.name = 'Date'
#*****************************************************************************\
#Dataframe YOTC All
df_yotc_all=df_yotc.reindex(date_index_all)
df_yotc_all.index.name = 'Date'
# #*****************************************************************************\
# #Dataframe MAC
# dm={'Clas':mac_clas,
# 'Depth':mac_depth,
# '1 Inv BL': mac_hght_1invBL,
# '2 Inv BL': mac_hght_2invBL,
# '1 Inv DL': mac_hght_1invDL,
# '2 Inv DL': mac_hght_2invDL,
# '1ra Inv': mac_hght_1inv,
# '2da Inv': mac_hght_2inv,
# 'Strg 1inv': mac_strg_1inv,
# 'Strg 2inv': mac_strg_2inv}

# df_mac = pd.DataFrame(data=dm,index=time_my)
# # Eliminate Duplicate Soundings

# dfm=df_mac.reset_index().drop_duplicates(cols='index',take_last=True).set_index('index')

# df_mac_final=dfm.reindex(date_index_all)
# df_mac_final.index.name = 'Date'
# # count_nan = len(df_my_final) - df_my_final.count()

# #*****************************************************************************\
#Dataframe MAC YOTC levels
dmy={'Clas':mac_y_clas,
'Depth':mac_y_depth,
# '1 Inv BL': mac_y_hght_1invBL,
# '2 Inv BL': mac_y_hght_2invBL,
# '1 Inv DL': mac_y_hght_1invDL,
# '2 Inv DL': mac_y_hght_2invDL,
'1ra Inv': mac_y_hght_1inv,
'2da Inv': mac_y_hght_2inv,
'Strg 1inv': mac_y_strg_1inv,
'Strg 2inv': mac_y_strg_2inv,
'Mix R 925':mixr_my,
'T 925':temp_my,
'Theta v 925':pot_temp_my,
'u 925':u_my,
'v 925':v_my,
'RH 925':relhum_my}

df_mac_y = pd.DataFrame(data=dmy,index=time_my)
# Eliminate Duplicate Soundings
dfmy=df_mac_y.reset_index().drop_duplicates(cols='index',take_last=True).set_index('index')

df_macyotc_final=dfmy.reindex(date_index_all)
df_macyotc_final.index.name = 'Date'
#*****************************************************************************\
#Saving CSV
path_data_save=base_dir+'/Dropbox/Monash_Uni/SO/MAC/Data/00 CSV/'

#df_yotc.to_csv(path_data_save + 'df_yotc925.csv', sep='\t', encoding='utf-8')
df_yotc_all.to_csv(path_data_save + 'df_yotc_all925.csv', sep='\t', encoding='utf-8')
# df_mac_final.to_csv(path_data_save + 'df_mac_final.csv', sep='\t', encoding='utf-8')
df_macyotc_final.to_csv(path_data_save + 'df_macyotc_final925.csv', sep='\t', encoding='utf-8')

#*****************************************************************************\

# #T1 = pd.read_csv(path_data_save + 'df_macyotc_final.csv', sep='\t')

# #Dataframe datos per level
# #nlevel=range(1,31)
# #Crear listas con variables y unirlas a dataframe 3D
# #t_list=temp.tolist()
# #u_list=u.tolist()
# #v_list=v.tolist()
# #q_list=q.tolist()

# #data={'temp':t_list, 'q':q_list,'u':u_list, 'v':v_list}
# #df=pd.DataFrame(data=data, index=date_range)


from django.db import models
# from ....card import card

from django.contrib.auth import get_user_model
User = get_user_model()

class Game(models.Model):

    # Each game is mapped to one user
    user = models.OneToOneField(User, on_delete = models.CASCADE)
    GAMESTATE = (
        ('battle','battle'),
        ('shop', 'shop'),
        ('event', 'event'),
        ('map', 'map'),
        ('reward','reward')
    )
    gamestate = models.CharField(max_length = 7, choices = GAMESTATE)
    curr_health = models.IntegerField()
    max_health = models.IntegerField()
    max_mana = models.IntegerField()
    gold = models.IntegerField()
    # array of card
    deck = models.JSONField(default = list)
    relics = models.JSONField(default = dict)

    # game.id is always == user.id
    def __str__(self):
        return str(self.id)