"""
@author: Ming Ming Zhang, mmzhangist@gmail.com

Utilities
"""

import time
import random
import numpy as np
import skimage.io, skimage.color, skimage.transform

############################################################
#  Image Preprocessing
############################################################

def load_image(image_path, mode):
    """
    Loads an image into RGB or gray.

    Parameters
    ----------
    image_path : string
        The image file path.

    Returns
    -------
    image : numpy array, [h, w, 3] for RGB or [h, w] for gray
        Loaded image.

    """
    image = skimage.io.imread(image_path)

    assert mode in ['gray', 'rgb']
    if mode == 'rgb':
        # convert to RGB if it is grayscale
        if image.ndim != 3:
            # print('%s was grayscale'%image_path)
            image = skimage.color.gray2rgb(image)
        # convert to RGB if it is RGBA
        if image.shape[-1] == 4:
            # print('%s was RGBA'%image_path)
            image = image[..., :3]

    else:
        # convert to grayscale if it is RGB
        if image.ndim != 2:
            # print('%s was RGB'%image_path)
            image = skimage.color.rgb2gray(image)

    return image

def resize_image(
        image,
        shortest_side=512,
        longest_side=1024,
        upscale_factor=1.0,
        mode='crop'
        ):
    """
    Resizes an image without changing the aspect ratio.

    Parameters
    ----------
    image : numpy array, [height, width, channels]
        The image needed to be resized.
    shortest_side : integer, optional
        The shortest side of the resized image. Note that it only applies to
        the 'crop' mode. The default is 512.
    longest_side : integer, optional
        The longest side of the resized image. Note that it only applies to the
        'pad_square' mode. The default is 1024.
    upscale_factor : float, optional
        The scale factor >= 1.0 to upscale the image. The default is 1.0.
    mode : string, optional
        The resizing method in {'none'', 'pad_square', 'crop}. The default is
        'crop'.
        * 'none' : [height, width, channels], no resizing nor padding applied.
        * 'pad_square' : [longest_side, longest_side, channels], padded with 0
        to keep the same aspect ratio.
        * 'crop' : [shorest_side, shortest_side, channels].

    Returns
    -------
    image : numpy array, [resized_h, resized_w, channels]
        The resized image depending on the mode with the same data type as the
        input image.
    window : tuple
        The corner coordinates (y1, x1, y2, x2) indicates the location of the
        resized image before padding.
    scale : float
        The scale factor from original to resized image, and used in the later
        resize_mask().
    padding : list
        The padding applied to the original image in the form of pixels
        [(top, bottom), (left, right), (0, 0)], and used in the later
        resize_mask().
    crop : tuple
        The coordinates (y1, x1, h, w) indicates the location of the
        cropped image over the resized image, and used in the later
        resize_mask().

    """
    assert mode in ['none', 'pad_square', 'crop']

    image_dtype = image.dtype
    h, w = image.shape[:2]
    window = (0, 0, h, w)
    scale = 1.0
    if image.ndim == 3:
        padding = [(0,0), (0,0), (0,0)]
    else:
        padding = [(0,0), (0,0)]
    crop = None

    if mode == 'none':
        return image, window, scale, padding, crop

    # upscale for all modes, scale >= 1.0
    scale = max(1.0, shortest_side / min(h, w))
    if scale < upscale_factor:
        scale = upscale_factor

    # downscale only if mode == 'pad_square', scale < 1.0
    if mode == 'pad_square':
        max_side = max(h, w)
        if round(max_side * scale) > longest_side:
            scale = longest_side / max_side

    # resize image using bilinear
    if scale != 1.0:
        image = skimage.transform.resize(
            image,
            output_shape=(round(h * scale), round(w * scale)),
            preserve_range=True,
            anti_aliasing=False)

    if mode == 'pad_square':
        h, w = image.shape[:2]
        top_pad = (longest_side - h) // 2
        bottom_pad = longest_side - h - top_pad
        left_pad = (longest_side - w) // 2
        right_pad = longest_side - w - left_pad
        if image.ndim == 3:
            padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
        else:
            padding = [(top_pad, bottom_pad), (left_pad, right_pad)]
        image = np.pad(image, padding)
        window = (top_pad, left_pad, h + top_pad, w + left_pad)

    else:
        h, w = image.shape[:2]
        # note that h (or w) - shortest_side maybe 0
        y = random.randint(0, (h - shortest_side))
        x = random.randint(0, (w - shortest_side))
        crop = (y, x, shortest_side, shortest_side)
        image = image[y:y + shortest_side, x:x + shortest_side]
        window = (0, 0, shortest_side, shortest_side)

    return image.astype(image_dtype), window, scale, padding, crop

############################################################
#  Dataset
############################################################

class Dataset(object):
    """
    Defines a dataset class.

    """

    def __init__(self):
        # image_info, list of dictionaries
        # {'id':image_id, 'path':path, 'class_name':class_name, ...}
        self.image_info = []

        # class_info, list of dictionaries
        # {'id':class_id, 'name':class_name}
        self.class_info = []

    def add_image(self, image_id, path, class_name, **kwargs):
        """
        Adds image information to the dataset.

        Parameters
        ----------
        image_id : string/integer
            The image ID.
        path : string
            The image path.
        class_name : string
            The corresponding class name.
        **kwargs : string/integer
            Additional information, e.g., image's height, width and so on.

        Returns
        -------
        None.

        """
        image_info = {'id':image_id, 'path':path, 'class_name':class_name, }
        image_info.update(kwargs)
        self.image_info.append(image_info)

    def add_class(self, class_id, class_name):
        """
        Adds class information to the dataset.

        Parameters
        ----------
        class_id : integer
            The class ID.
        class_name : string
            The corresponding class name. Augmented class names have the forms:
                * name_rot_n : rotate n degree where n in [90,180,270]
                * name_flip_lr : flip horizontally
                * name_flip_ud : flip vertically

        Returns
        -------
        None.

        """
        for info in self.class_info:
            if info['id'] == class_id:
                # skip if class_id already existed
                return
        self.class_info.append({'id':class_id, 'name':class_name})

    def prepare(self):
        """
        Prepares the dataset and needs to be called before using it.

        Returns
        -------
        None.

        """
        # info from class_info and image_info
        self.num_classes = len(self.class_info)
        self.class_ids = np.arange(self.num_classes)
        self.class_names = [
            a_class_info['name'] for a_class_info in self.class_info]
        self.num_images = len(self.image_info)
        self.image_ids = np.arange(self.num_images)

    def get_ds_info(self):
        """
        Gets the dataset information.

        Returns
        -------
        class_names : list
            Each element is a name corresponding to the class id.

        """
        print('There are %d images and %d classes.' \
              % (self.num_images, self.num_classes))
        return self.class_names

    def get_img_info(self, image_id):
        """
        Gets an image information.

        Parameters
        ----------
        image_id : integer
            An image id from image_ids.

        Returns
        -------
        image_info : dictionary
            The image information with keys 'id', 'path' and so on; see
            add_image().

        """
        return self.image_info[image_id]

    def image_name2id(self, image_id_str):
        """
        Gets the image id from the name of image.

        Parameters
        ----------
        image_id_str : string
            The name of image, image_info['id'].

        Returns
        -------
        image_id : integer
            The corresponding image id.

        """
        for image_id in self.image_ids:
            if self.get_img_info(image_id)['id'] == image_id_str:
                return image_id

    def class_name2id(self, class_name):
        """
        Gets the class id from the name of class.

        Parameters
        ----------
        class_name : string
            The given class name.

        Returns
        -------
        integer
            The corresponding class id.

        """
        for class_id in self.class_ids:
            if class_name == self.class_info[class_id]['name']:
                return self.class_info[class_id]['id']

    def class2images(self, class_id):
        """
        Get image ids from the class id. It needs to be re-defined in a
        subclass; otherwise it won't be efficient.

        Parameters
        ----------
        class_id : integer
            The given class id.

        Returns
        -------
        image_ids : list
            The set of image ids for the given class id.

        """
        image_ids = []
        for image_id in self.image_ids:
            class_name = self.get_img_info(image_id)['class_name']
            if self.class_name2id(class_name) == class_id:
                image_ids.append(image_id)
        return image_ids

    def load_data(
            self,
            image_id,
            img_mode,
            shortest_side=84,
            longest_side=84,
            upscale_factor=1.0,
            resize_mode='pad_square',
            verbose=0
            ):
        """
        Loads preprocessed image, class ids and other information.

        Parameters
        ----------
        image_id : integer
            The image id from image_ids.
        img_mode : string
            Process image to either in 'rgb' or 'gray'.
        shortest_side, longest_side, upscale_factor, resize_mode :
            See resize_image().
        verbose : binary, optional
            Whether to print out the preprocessing time. The default is 0.

        Returns
        -------
        image1 : numpy array, [resized_h, resized_w, 3] for RGB or
        [resized_h, resized_w] for gray
            The preprocessed image.
        class_id : integer
            The class id for the image.
        cache : list
            Includes image path, class name, window, scale, padding, crop,
            original image shape and preprocessed image shape.

        """
        # load image
        image_path = self.get_img_info(image_id)['path']
        t1 = time.time()
        image = load_image(image_path, img_mode)
        t2 = time.time()
        if verbose:
            print('loading image: %fs' %(t2-t1))
        image_shape = image.shape

        # resize image
        t1 = time.time()
        image1, window, scale, padding, crop = resize_image(
            image,
            shortest_side,
            longest_side,
            upscale_factor,
            resize_mode)
        t2 = time.time()
        if verbose:
            print('resize mode:', resize_mode)
            print('resizing image: %fs' %(t2-t1))

        # augment if needed
        class_name = self.get_img_info(image_id)['class_name']
        if 'rot' in class_name.split('_'):
            angle = int(class_name.split('_')[-1])
            k = angle / 90
            t1 = time.time()
            # image1 = skimage.transform.rotate(image1, angle)
            image1 = np.rot90(image1, k)
            t2 = time.time()
            if verbose:
                print('rotating image: %fs' %(t2-t1))
        elif 'flip' in class_name.split('_'):
            if 'lr' in class_name.split('_'):
                t1 = time.time()
                image1 = np.fliplr(image1)
                t2 = time.time()
            if 'ud' in class_name.split('_'):
                t1 = time.time()
                image1 = np.flipud(image1)
                t2 = time.time()
            if verbose:
                print('fliping image: %fs' %(t2-t1))

        cache = (
            image_path,
            class_name,
            window,
            scale,
            padding,
            crop,
            image_shape,
            image1.shape)
        class_id = self.class_name2id(class_name)
        return image1, class_id, cache


#! /usr/bin/env python
"""
Script to copy a column between MS files
"""
import argparse
from argparse import RawTextHelpFormatter
import casacore.tables as pt
import numpy
import sys
import os

def copy_column_to_ms(ms, inputcol, outputcol, ms_from=None, use_compression=False):
    """
    Copies one column to another, within an MS file or between two MS files

    Parameters
    ----------
    ms : str
        MS file receiving copy
    inputcol : str
        Column name to copy from
    outputcol : str
        Column name to copy to
    ms_from : str, optional
        MS file to copy from. If None, the column is copied internally

    """
    t = pt.table(ms, readonly=False, ack=False)
    if ms_from is not None:
        tf = pt.table(ms_from, readonly=False, ack=False)
        data = tf.getcol(inputcol)
        desc = tf.getcoldesc(inputcol)
    else:
        data = t.getcol(inputcol)
        desc = t.getcoldesc(inputcol)

    # Add the output column if needed
    if outputcol not in t.colnames():
        if use_compression:
            # Set DyscoStMan to be storage manager for DATA and WEIGHT_SPECTRUM
            # We use a visibility bit rate of 16 and truncation of 1.5 sigma to keep the
            # compression noise below ~ 0.01 mJy, as estimated from Fig 4 of
            # Offringa (2016). For the weights, we use a bit rate of 12, as
            # recommended in Sec 4.4 of Offringa (2016)
            desc['name'] = outputcol
            dmi = {
                'SPEC': {
                    'dataBitCount': numpy.uint32(16),
                    'distribution': 'TruncatedGaussian',
                    'distributionTruncation': 1.5,
                    'normalization': 'RF',
                    'weightBitCount': numpy.uint32(12)},
                'NAME': '{}_dm'.format(outputcol),
                'SEQNR': 1,
                'TYPE': 'DyscoStMan'}
            desc['option'] = 1 # make a Direct column
            t.addcols(desc, dmi)
        else:
            desc['name'] = outputcol
            t.addcols(desc)

    if use_compression:
        # Replace flagged values with NaNs before compression
        flags = t.getcol('FLAG')
        flagged = numpy.where(flags)
        data[flagged] = numpy.NaN

    t.putcol(outputcol, data)
    t.flush()
    t.close()

def copy_column_to_bands(mslist, ms_from, inputcol, outputcol):
    """
    Copies one column from an MS file to multiple MS files (bands)

    Parameters
    ----------
    mslist : list
        MS files receiving copy
    ms_from : str
        MS file to copy from
    inputcol : str
        Column name to copy from
    outputcol : str
        Column name to copy to

    """
    datain = pt.table(ms_from)
    data = datain.getcol(inputcol, nrow=1)
    numberofchans = numpy.int(numpy.shape(data)[1])
    chanperms = numberofchans/numpy.int(len(mslist))

    for ms_id, ms in enumerate(mslist):
        if os.path.isdir(ms):
            data = datain.getcolslice(inputcol, [chanperms*ms_id,0], [(chanperms*(ms_id+1))-1,3])
            dataout = pt.table(ms, readonly=False)
            dataout.putcol(outputcol, data)
            dataout.flush()
            dataout.close()

def copy_column_from_bands(mslist, ms_to, inputcol, outputcol):
    """
    Copies one column from multiple MS files (bands) to a single MS file

    Note: the bands are assumed to be ordered by frequency, with a nonexisting
    file (e.g., 'dummy.ms') denoting missing bands

    Parameters
    ----------
    mslist : list
        MS files to copy from
    ms_to : str
        MS file receiving copy
    inputcol : str
        Column name to copy from
    outputcol : str
        Column name to copy to

    """
    dataout = pt.table(ms_to, readonly=False)
    data = dataout.getcol(outputcol, nrow=1)
    numberofchans = numpy.int(numpy.shape(data)[1])
    chanperms = numberofchans/numpy.int(len(mslist))

    for ms_id, ms in enumerate(mslist):
        if os.path.isdir(ms):
            datain = pt.table(ms, readonly=True)
            data = datain.getcol(inputcol)
            dataout.putcolslice(outputcol, data, [chanperms*ms_id,0], [(chanperms*(ms_id+1))-1,3])
            datain.close()
    dataout.flush()
    dataout.close()

def main(ms_from, ms_to, column_from, column_to, do_copy=True, use_compression=False):
    """
    Copy a column between MS files

    Parameters
    ----------
    ms_from : str
        Name of MS file to copy from
    ms_to : str or list
        Name of MS file or list of MS files to copy to. May be given as a list
        or as a string (e.g., '[ms1, ms2]'
    column_from : str
        Name of column to copy from
    column_to : str
        Name of column to copy to
    do_copy : bool, optional
        If False, the copy is NOT done (used to skip a copy step in a pipeline)
    use_compression : bool, optional
        If True, use Dysco compression

    """
    if type(do_copy) is str:
        if do_copy.lower() == 'true':
            do_copy = True
        else:
            do_copy = False
    if type(use_compression) is str:
        if use_compression.lower() == 'true':
            use_compression = True
        else:
            use_compression = False

    if not do_copy:
        print('Copy skipped (do_copy = False)')
        return

    if type(ms_to) is str:
        if '[' in ms_to:
            ms_to = ms_to.strip('[]').split(',')
            ms_to = [m.strip() for m in ms_to]

    if type(ms_from) is str:
        if '[' in ms_from:
            ms_from = ms_from.strip('[]').split(',')
            ms_from = [m.strip() for m in ms_from]

    if type(ms_to) is list and type(ms_from) is list:
        print('ERROR: ms_from and ms_to cannot both be lists')
        sys.exit(1)

    if type(ms_to) is list:
        # List means call copy_column_to_bands()
        copy_column_to_bands(ms_to, ms_from, column_from, column_to)
    elif type(ms_from) is list:
        # List means call copy_column_from_bands()
        copy_column_from_bands(ms_from, ms_to, column_from, column_to)
    else:
        if ms_to == ms_from:
            ms_from = None
        copy_column_to_ms(ms_to, column_from, column_to, ms_from, use_compression)

if __name__ == '__main__':
    descriptiontext = "Copy a column between MS files.\n"

    parser = argparse.ArgumentParser(description=descriptiontext, formatter_class=RawTextHelpFormatter)
    parser.add_argument('ms_from', help='name of the MS file to copy from')
    parser.add_argument('ms_to', help='name of the MS file to copy to')
    parser.add_argument('column_from', help='name of the column to copy from')
    parser.add_argument('column_to', help='name of the column to copy to')
    parser.add_argument('do_copy', help='Copy is done only if True')
    args = parser.parse_args()

    main(args.ms_from, args.ms_to, args.column_from, args.column_to,
        do_copy=args.do_copy)


# coding: utf-8

# In[1]:

import numpy as np
import csv
import pandas

csv = pandas.read_csv(filepath_or_buffer="DataDaily.csv", sep=";")
csv

# In[2]:

mean = csv.mean()

# In[3]:

covmatr = csv.cov()

# In[4]:

matrixCov = covmatr.as_matrix()

matrixCov.shape
eigenvalues, eigenvectors = np.linalg.eigh(matrixCov)

# In[7]:

size = eigenvalues.size

eigenvalues[size - 3:size]

# In[9]:

#соответсвующие собственные векора eignvertor[]

b3 = eigenvectors[:, size-3]
lambda3 = eigenvalues[size-3]

b2 = eigenvectors[:, size-2]
lambda2 = eigenvalues[size-2]

b1 = eigenvectors[:, size-1]
lambda1 = eigenvalues[size-1]

print(b1)
print(lambda1)
print("------------")
print(b2)
print(lambda2)
print("------------")
print(b3)
print(lambda3)
print("------------")

# In[10]:

fac1 = [-2.45, -1.63, -0.82, 0, 0.82, 1.63, 2.45]
prob1 = [0.016, 0.094, 0.234, 0.313, 0.234, 0.094, 0.016]

fac2 = [-2, -1.000, 0.000 ,1.000 ,2.000]
prob2 = [0.062, 0.250, 0.370, 0.250, 0.062]

fac3 = [-1.410, 0.000, 1.410]
prob3 = [0.25, 0.5, 0.25]

# In[13]:

#iter method
import scipy.stats as stats

delta = 0.1
eps = 0.001

def iterStep(x, n, val):
    flag = False
    x[0] = val
    x[1] = invF(2*F(x[0]))
    for i in range(1, n-1):
        odd(i, x)
        even(i, x, flag)
    odd(n-1, x)
    print(G(x,n))

def IterMet(n, delta):
    x = np.zeros(2*n-1)
    flag = False
    iterStep(x, n, invF(2 ** (-n)))
    valueG = G(x, n)
    prevG = valueG
    while (abs(valueG) > eps or delta < 0.001):
        valueG = G(x, n)
        if (prevG*valueG < 0):
            delta /= 2
        prevG = valueG
        if (valueG > 0):
            iterStep(x, n, x[0] + delta)
        else:
            iterStep(x, n, x[0] - delta)
    print("Итоговая точность")
    print(G(x,n))
    print()
    return x

def invF(x):
    return stats.norm.ppf(x)

def F(x):
    return stats.norm.cdf(x)

def odd(i, x):
    x[2*i] =  2 * x[2*i-1] - x[2*i - 2]

def even(i, x, flag):
    value = -F(x[2*i-1]) + 2 * F(x[2*i])
    if (value > 1):
        print("error")
        print(2 * i + 1)
        print(value)
        flag = True
    if (value < 0):
        print("error")
        print(2 * i + 1)
        print(value)
        flag = True
    x[2*i + 1] = invF(value)

def G(x, n):
    return F(x[2*n-3]) - 2*F(x[2*n-2]) + 1

x = IterMet(2, delta)
a3 = x[:3:2]

x = IterMet(4, delta)
a2 = x[:7:2]

x = IterMet(6, delta)
a1 = x[:11:2]

print()
print(a1)
print(a2)
print(a3)

# In[14]:

def probV(a, n):
    prob = np.zeros(n+1)
    prob[0] = F(a[0])
    for i in range(1,n):
        prob[i] = F(a[i]) - F(a[i-1])
    prob[n] = 1 - F(a[n-1])
    return prob

# In[15]:

probvecUpd1 = probV(a1, 6)

# In[16]:

probvecUpd2 = probV(a2, 4)

# In[17]:

probvecUpd3 = probV(a3, 2)

# In[18]:

import scipy.integrate as integrate
from scipy.integrate import quad

def integrand(x):
    return x*stats.norm.pdf(x)

def findvalue(n, a, b):
    x = np.zeros(n+1)
    x[0] , err = quad(integrand, a=-np.inf, b=a[0]) / b[0]
    for i in range (1,n):
        x[i], err =  quad(integrand, a=-a[i-1], b=a[i]) / b[i]
    x[n] , err =  quad(integrand, b=np.inf, a=a[n-1]) / b[n]
    return x

val1 = findvalue(6, a1, probvecUpd1)
val2 = findvalue(4, a2, probvecUpd2)
val3 = findvalue(2, a3, probvecUpd3)

val1 = val1.tolist()
probvecUpd1 = probvecUpd1.tolist()

val2 = val2.tolist()
probvecUpd2 = probvecUpd2.tolist()

val3 = val3.tolist()
probvecUpd3 = probvecUpd3.tolist()

print(val1)
print(probvecUpd1)
print()
print(val2)
print(probvecUpd2)
print()
print(val3)
print(probvecUpd3)

# In[19]:

csvmodel = pandas.read_csv(filepath_or_buffer="2k15day.csv", sep=";", index_col=0)
csvmodel

# In[20]:

times = csvmodel.index.size - 1
times

# In[21]:

import matplotlib.pyplot as plt
matrModel = csvmodel.as_matrix()

d = np.zeros(matrModel[0].size)
for i in range(0, len(d)):
    d[i] = 1000 / matrModel[0][i]
d

# In[24]:

P = np.zeros(len(matrModel))
deltaP = np.zeros(len(P) - 1)

for idx in range(0, len(matrModel)):
    for pos in range(0, len(matrModel[idx])):
        P[idx] += matrModel[idx][pos] * d[pos]

for idx in range(0, len(P) - 1):
    deltaP[idx] = round(P[idx+1] - P[idx])

deltaP

df = pandas.DataFrame({'DeltaP' : deltaP}, index=csvmodel.index[:times])

# In[25]:

#монте-карло
import math as math

def monteF():
    w1 = np.random.normal()
    w2 = np.random.normal()
    w3 = np.random.normal()
    arr = np.zeros(mean.size)
    for indx in range(0, mean.size):
        arr[indx] = montefunc(indx, w1, w2, w3)
    return arr

def montefunc(idx, w1, w2, w3):
    return math.exp(mean[idx] + math.sqrt(lambda1) * b1[idx] * w1 +
                   math.sqrt(lambda2) * b2[idx] * w2 +
                   math.sqrt(lambda3) * b3[idx] * w3)

# In[26]:

def func(prob1, fac1):
    rand = np.random.uniform()
    sum1 = 0
    for prob in prob1:
        sum1 += prob
        if rand < sum1:
            return fac1[prob1.index(prob)]
    return fac1[len(prob1) - 1]

# In[27]:

def funcScenario(prob1, fac1, prob2, fac2, prob3, fac3):
    w1 = func(prob1, fac1)
    w2 = func(prob2, fac2)
    w3 = func(prob3, fac3)
    arr = np.zeros(mean.size)
    for indx in range(0, mean.size):
        arr[indx] = montefunc(indx, w1, w2, w3)
    return arr

# In[28]:

#for Var
count = 500
shortfall = np.zeros(times)
alpha = 0.05
p =round(count * alpha)

varScenario = np.zeros(times)
for j in range(0,times):
    valueofP = []
    for i in range(0, count):
        valueofP.append(funcScenario(prob1, fac1,prob2,fac2,prob3,fac3))
    startv = matrModel[j] * d

    startp = np.dot(startv, np.ones(matrModel[j].size))

    value = np.zeros(count)

    for i in range(0, count):
        value[i] = np.dot(valueofP[i], startv)

    value.sort()
    varScenario[j] = value[p] - startp
    for i in range(0, p):
        shortfall[j] += value[i] - startp
    shortfall[j] /= p

# In[29]:

#for Var Updated
countf2 = 500

shortfallUpd = np.zeros(times)

varScenarioUpd = np.zeros(times)
for j in range(0,times):
    valueofP = []
    for i in range(0, count):
        valueofP.append(funcScenario(probvecUpd1, val1,probvecUpd2,val2,probvecUpd3,val3))
    startv = matrModel[j] * d

    startp = np.dot(startv, np.ones(matrModel[j].size))

    value = np.zeros(count)

    for i in range(0, count):
        value[i] = np.dot(valueofP[i], startv)

    value.sort()
    varScenarioUpd[j] = value[p] - startp
    for i in range(0, p):
        shortfallUpd[j] += value[i] - startp
    shortfallUpd[j] /= p

# In[30]:

var1 = -varScenario
var2 = -varScenarioUpd
deltap1 = -deltaP
for i in range(0, len(deltap1)):
    if deltap1[i] < 0:
        deltap1[i] = 0
title = "Var " + str(alpha)
df1 = pandas.DataFrame({'varScenario' : var1, 'varScenarioUpd' : var2, 'Loses' : deltap1 }, index=csvmodel.index[:times])
df1
df1.plot(kind="line", title=title)
plt.show()

# In[31]:

var1sh = -shortfall
var2sh = -shortfallUpd
deltap1 = -deltaP
for i in range(0, len(deltap1)):
    if deltap1[i] < 0:
        deltap1[i] = 0

df1 = pandas.DataFrame({'shortfall' : var1sh, 'shorfallUpd' : var2sh, 'Loses' : deltap1 }, index=csvmodel.index[:times])
df1
df1.plot(kind="line", title="Shortfall " + str(alpha))
plt.show()

# In[32]:

m2 =df1.shorfallUpd.mean()

# In[33]:

m1 = df1.shortfall.mean()

# In[34]:

dict1 = {'mean' : [m1, m2]}

# In[35]:

df3 = pandas.DataFrame(dict1, index=["shortfall", "shortfallUpd"] )
df3

# In[ ]:


import os
import json
from PIL import Image

# read configurations
with open('config.json', 'r') as file:
    data = json.load(file)

config = data[0]

def resize_images_with_original_rotation(input_dir, output_dir, new_size):
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    for filename in os.listdir(input_dir):
        if filename.endswith(('.jpg', '.jpeg', '.png', '.gif')):
            image_path = os.path.join(input_dir, filename)
            image = Image.open(image_path)

            original_rotation = 0
            if hasattr(image, '_getexif') and image._getexif() is not None:
                exif_data = image._getexif()
                if 274 in exif_data:
                    orientation = exif_data[274]
                    if orientation == 3:
                        original_rotation = 180
                    elif orientation == 6:
                        original_rotation = 270
                    elif orientation == 8:
                        original_rotation = 90

            image.thumbnail(new_size, Image.ANTIALIAS)
            image = image.rotate(original_rotation, expand=True)
            output_path = os.path.join(output_dir, filename)
            image.save(output_path)
            print("Resized image with original rotation saved to:", output_path)

# execute
source =  config['sourceDirectory']
destination =  config['destinationDirectory']
width = int(config['resizeWidth'])
height = int(config['resizeHeight'])
new_size = (width, height)

resize_images_with_original_rotation(source, destination, new_size)


import numpy as np
import sklearn.neighbors
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.patches
import scipy
import re
import subprocess
import itertools

import pdb

###############################################################################
#
# SIMPLICIAL COMPLEXES
#
###############################################################################

def triangle_cliques(connections):
    """
    Returns all 3-cliques implied by the upper-triangular sparse connections
    matrix.
    """
    triangles = []
    for i in range(connections.shape[0]):
        adjacency = connections.getrow(i)
        # ignore self loops
        adjacency[0,i] = False
        neighbors = np.nonzero(adjacency)[1].tolist()
        while neighbors:
            j = neighbors.pop(0)
            for k in neighbors:
                if connections[j,k]:
                    triangles.append([i,j,k])

    return triangles

def kneighbors_2skeleton(data, k):
    """
    Builds the 2-skeleton of the data by connecting each point to its k nearest
    neighbors, then making triangles out of sets of 3 mutually-connected
    vertices. Returns a list of edges and a list of triangles.
    """
    # compute the k nearest neighbors
    knn = sklearn.neighbors.kneighbors_graph(data, k)
    # make the connections symmetric, and return only the upper triangle
    connections = scipy.sparse.triu((knn + knn.T).astype(bool)).tocsr()

    # find mutually-connected triangles
    triangles = triangle_cliques(connections)

    # get the list of edges
    u,v = np.nonzero(connections)
    edges = zip(u.tolist(), v.tolist())
    # remove self edges
    edges = [e for e in edges if e[0] != e[1]]

    return edges, triangles

def simplify_triangle_edges(edges, tris):
    simplified_edges = list(edges)
    for tri in tris:
        for p,q in itertools.combinations(tri, 2):
            if (p,q) in simplified_edges:
                simplified_edges.remove((p,q))
            if (q,p) in simplified_edges:
                simplified_edges.remove((q,p))
    return simplified_edges

def plot_2d_complex(data, edges, tris):
    """
    Plots a complex whose points are embedded in 2d Euclidean space.
    """
    # draw the points
    plt.scatter(data[:,0], data[:,1])
    # now draw each edge
    for u,v in edges:
        x = data[(u,v),0]
        y = data[(u,v),1]
        plt.plot(x, y, color='black', alpha=0.5)

    for u,v,w in tris:
        points = data[(u,v,w),:]
        poly = matplotlib.patches.Polygon(points, facecolor='red', alpha=0.2,
                color='red')
        plt.gca().add_patch(poly)

    plt.show()

###############################################################################
#
# GRAPH FUNCTIONS
#
###############################################################################

def geodesic_distance(plex, root=None):
    """
    Given a simplicial complex in the form of a networkx graph, computes the
    geodesic distance to each node from a given root node. If the root is not
    specified, an arbitrary extreme root is found by eccentricity. Returns a
    list of distances.

    Assumes the graph is connected.
    """
    if root is None:
        # compute the eccentricity of each node
        max_distances = nx.eccentricity(plex)
        root = max(max_distances.items(), key=lambda x: x[1])[0]

    # compute the distance from the root to each node
    distances = nx.shortest_path_length(plex, root)
    return [distances[v] for v in sorted(plex.nodes())]

###############################################################################
#
# FUNCTIONS ON REEB GRAPHS
#
###############################################################################

def is_regular(graph, node):
    """
    Determines whether a node in the graph is regular.
    A node is regular if:
        1) It is of degree 2.
        2) It is the only path between its neighbors.
    """
    if nx.degree(graph, node) != 2:
        return False

    # get the parents, of which there are two
    u,v = nx.neighbors(graph, node)

    n_simple_paths = 0
    for path in nx.all_simple_paths(graph, u, v):
        n_simple_paths += 1
        if n_simple_paths > 1:
            return False
    else:
        return True

def contract(reeb_graph):
    """
    Given an augmented reeb graph, contracts it, returning a graph where each
    edge has a "members" property listing the nodes it contains.
    """
    g = reeb_graph.copy()
    # give every edge the members property
    for u,v in g.edges_iter():
        g[u][v]['members'] = set()

    # now reduce each that is regular
    for node in g.nodes():
        if is_regular(g, node):
            # get the neighbors
            u,v = nx.neighbors(g, node)
            # link them
            g.add_edge(u,v)

            # record the members of the edges
            members = set([node])
            members = members | g.edge[u][node]['members']
            members = members | g.edge[v][node]['members']
            g.edge[u][v]['members'] = members

            # delete the original node
            g.remove_node(node)

    return g

###############################################################################
#
# RANDREEB INTERFACE
#
###############################################################################

def randreeb(heights, edges, tris):
    """
    Interfaces with the randreeb program to compute the reeb graph of a
    function defined on the 2-skeleton of a simplicial complex.
    """
    import os
    module_path = os.path.dirname(os.path.abspath(__file__))
    os.chdir(module_path)
    write_OFF("complex.off", heights, edges, tris)
    # call randreeb
    output = subprocess.check_output(["./randreeb/ReebGraph",
            "-z", "-a", "complex.off"])
    # read the output
    values, edges = read_OFF("out_aug.off")
    return edges

###############################################################################
#
# OFF FILE HANDLING
#
###############################################################################

def read_OFF(filepath):
    """
    Reads an OFF file, returning a list of values on the vertices, and a list
    of faces (edges are ignored).
    """
    with open(filepath,"r") as f:
        # read until we get to the first line containing the counts
        line = f.readline()
        pattern = "^(\d+) (\d+) (\d+)"
        matches = re.findall(pattern, line)
        while not matches:
            line = f.readline()
            matches = re.findall(pattern, line)
        n_vertices, n_faces, _ = [int(x) for x in matches[0]]

        # read the vertices in, taking only the last number of each line
        values = []
        for i in range(n_vertices):
            line = f.readline().strip()
            vertex = [float(x) for x in line.split()]
            values.append(vertex[-1])

        # read the faces in
        faces = []
        for i in range(n_faces):
            line = f.readline().strip()
            face = [int(x) for x in line.split()]
            dim = face[0]
            if face[-4:] == [1,0,0,1] or True:
                faces.append(face[1:dim+1])

        return values, faces

def write_OFF(filepath, values, edges, faces):
    """
    Writes an OFF file that is readable by the above.
    """
    with open(filepath,"w+") as f:
        f.write("OFF\n")
        f.write("{} {} 0\n".format(len(values), len(edges)+len(faces)))
        for v in values:
            f.write("0 0 {}\n".format(v))

        for x in edges+faces:
            f.write("{} ".format(len(x)))
            f.write(" ".join([str(y) for y in x]))
            f.write("\n")


from django.urls import path

from user import views
urlpatterns = [
    path("signup",views.signup,name="signup"),
    path("signin",views.signin,name="signin"),
    path("signout",views.signout,name="signout"),
    path("profile",views.profile,name="profile"),
    path("add_address",views.add_address,name="add_address")
]


"""
Train base model. This model will be used to continue training.
"""
from __future__ import print_function, division

import warnings
warnings.filterwarnings('ignore')

import os
import sys
import platform
from pathlib import Path
import psutil
import argparse
from datetime import datetime
from time import time
from pprint import pprint, pformat
from collections import OrderedDict
from glob import glob

import sklearn
import numpy as np
import pandas as pd

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.externals import joblib
from sklearn.metrics import r2_score, mean_absolute_error, median_absolute_error, explained_variance_score
from scipy.stats import spearmanr

import keras as ke
from keras import backend as K

from keras.models import Sequential, Model, model_from_json, model_from_yaml, load_model
from keras.optimizers import SGD, Adam, RMSprop, Adadelta
from keras.utils import np_utils, multi_gpu_model, plot_model
from keras.callbacks import Callback, ModelCheckpoint, CSVLogger, ReduceLROnPlateau, EarlyStopping

import lightgbm as lgb

SEED = 42
t_start = time()

# File path
file_path = Path(__file__).resolve().parent

# Utils
import utils

utils_path = file_path / '../../utils'
sys.path.append(str(utils_path))
#import utils
from classlogger import Logger
import ml_models
from ml_models import r2_krs

# Import custom callbacks
keras_contrib = '/vol/ml/apartin/projects/keras-contrib/keras_contrib/callbacks'
sys.path.append(keras_contrib)
from cyclical_learning_rate import CyclicLR

# Path
PRJ_NAME = Path(file_path.name)/'transfer_learn_keras'
OUTDIR = file_path / '../../out/' / PRJ_NAME

def parse_args(args):
    parser = argparse.ArgumentParser(description="Transfer learning.")

    # Input data
    parser.add_argument('-bs_dir', '--base_model_dir', default=None, type=str, help='Path to base (pre-trained) model dir.')
    parser.add_argument('--tl_src', default='PDM', type=str, choices=['CCLE', 'CTRP', 'gCSI', 'GDSC', 'NCI60', 'all'],
                        help='Data name for which we apply transfer learning (default: PDM).')
    parser.add_argument('--tl_method', default='fe', type=str, choices=['ft', 'fe'], help="'ft': finetune, 'fe': feature extractor")

    parser.add_argument('--trial', default=0, type=int, help='PDM trial (there are 10 trial, in each trial 10 cv splits; default: 0).')

    parser.add_argument('--drg_subset', default='all', choices=['pdm', 'common', 'all'], help='Drug subset to use for training (default: all).')

    parser.add_argument('-ml', '--model_name', type=str, default='nn_reg0', help="Model to use train baseline NN model (default: 'nn_reg0').")
    parser.add_argument('-ep', '--epochs', default=50, type=int, help='Epochs (default: 50).')
    parser.add_argument('-b', '--batch_size', default=32, type=float, help='Batch size (default: 32).')
    parser.add_argument('--dr_rate', default=0.2, type=float, help='Dropout rate (default: 0.2).')

    parser.add_argument('--opt', default='clr_exp', type=str, choices=['sgd', 'adam', 'clr_trng1', 'clr_trng2', 'clr_exp'], help="Optimizer name (default: 'clr_exp').")
    parser.add_argument('--base_lr', type=float, default=1e-4, help='Base lr for cycle lr.')
    parser.add_argument('--max_lr', type=float, default=1e-3, help='Max lr for cycle lr.')
    parser.add_argument('--gamma', type=float, default=0.999994, help='Gamma parameter for learning cycle LR (default: 0.999994).')

    parser.add_argument('-kp_to', '--keep_up_to', type=str, default='a4', help='Feature extraction (how many layers to keep).')
    parser.add_argument('-frz_to', '--freeze_up_to', type=str, default='drp4', help='Finetuning (until what layer to freeze the network).')

    parser.add_argument('--skp_ep', type=int, default=3, help='Number of epochs to skip when plotting training curves.')
    parser.add_argument('--n_jobs', default=8, type=int, help='Number of cpu workers (default: 4).')
    args = parser.parse_args(args)
    return args

def create_outdir(outdir, args):
    t = datetime.now()
    t = [t.year, '-', t.month, '-', t.day, '_', 'h', t.hour, '-', 'm', t.minute]
    t = ''.join([str(i) for i in t])

    l = [args['bs_src'] + '_to_'+args['tl_src']] + [args['tl_method']] + [args['opt']] +  ['ep'+str(args['epochs'])] + ['drp'+str(args['dr_rate'])]

    name_sffx = '.'.join( l )
    outdir = Path(outdir) / (name_sffx + '_' + t)
    os.makedirs(outdir)
    return outdir

def run(args):
    base_model_dir = Path(args['base_model_dir'])
    tl_src = args['tl_src']
    tl_method = args['tl_method']
    trial = args['trial']

    ccl_fea_list = ['geneGE']
    drg_fea_list = ['DD']
    drg_subset = args['drg_subset']
    fea_sep = '_'

    model_name = args['model_name']
    epochs = args['epochs']
    batch_size = args['batch_size']
    dr_rate = args['dr_rate']
    opt_name = args['opt']
    base_lr = args['base_lr']
    max_lr = args['max_lr']
    gamma = args['gamma']
    skp_ep = args['skp_ep']

    n_jobs = args['n_jobs']
    verbose = True

    # Extract data sources used for base model training and transfer learning
    bs_src = str(base_model_dir).split('/')[-1].split('.')[0]
    args['bs_src'] = bs_src

    # Split of the dataset used for TRANSFER LEARNING
    bs_fold = 0

    # Transfer learning parameters
    keep_up_to = args['keep_up_to'] # feature extractor
    freeze_up_to = args['freeze_up_to'] # finetune

    # =====================================================
    #       Logger
    # =====================================================
    run_outdir = create_outdir(OUTDIR, args)
    lg = Logger(run_outdir/'logfile.log')
    lg.logger.info(f'File path: {file_path}')
    lg.logger.info(f'\n{pformat(args)}')

    # Dump args to file
    utils.dump_dict(args, run_outdir/'args.txt')

    # =====================================================
    #       Some global NN configs
    # =====================================================
    if opt_name == 'clr_trng1':
        clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, mode='triangular')
    elif opt_name == 'clr_trng2':
        clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, mode='triangular2')
    elif opt_name == 'clr_exp':
        clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, mode='exp_range', gamma=gamma) # 0.99994; 0.99999994; 0.999994

    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.75, patience=20, verbose=1, mode='auto',
                                  min_delta=0.0001, cooldown=3, min_lr=0.000000001)
    early_stop = EarlyStopping(monitor='val_loss', patience=100, verbose=1)

    # =====================================================
    #       Load data for Transfer Learning
    # =====================================================
    data = utils.load_data(file_path, src=tl_src,
            ccl_fea_list=ccl_fea_list, drg_fea_list=drg_fea_list,
            drg_subset=drg_subset,
            fea_sep=fea_sep, logger=lg.logger)

    if tl_src == 'PDM':
        main_folds_dir = Path(file_path/f'../../data/yitan/PDM_10Fold_Partition/trial_{trial}/')
        matched_to = bs_src
    else:
        main_folds_dir = Path(file_path/f'../../data/yitan/CCL_10Fold_Partition/{tl_src}/')
        matched_to = None

    # These lists contain scores for all folds
    all_bs_lgbm_scores = []
    all_bs_rf_scores = []
    #all_bs_nn_scores = []
    all_tl_scores = []

    # These lists contain preds for all folds
    all_bs_lgbm_preds = []
    all_bs_rf_preds = []
    #all_bs_nn_preds = []
    all_tl_preds = []

    for tl_fold in range(10):
        lg.logger.info('\n{}'.format('=' * 70))
        lg.logger.info('Start fold {}'.format(tl_fold))
        ids_path = main_folds_dir/f'cv_{tl_fold}'
        data_tr, data_vl, data_te = utils.get_splits_per_fold(data, ids_path, logger=None)

        xtr, ytr, mtr = utils.extract_data(data_tr, fea_list = ccl_fea_list+drg_fea_list, matched_to=matched_to)
        xvl, yvl, mvl = utils.extract_data(data_vl, fea_list = ccl_fea_list+drg_fea_list, matched_to=matched_to)
        xte, yte, mte = utils.extract_data(data_te, fea_list = ccl_fea_list+drg_fea_list, matched_to=matched_to)

        # Create dir to save results for the current fold
        fold_outdir = run_outdir/f'cv{tl_fold}'
        os.makedirs(fold_outdir)

        # =====================================================
        #       Scale (CV can start here) --> TODO: don't scale embeddeings
        # =====================================================
        scalerpath = base_model_dir/f'cv{bs_fold}'/'scaler.pkl'
        scaler = joblib.load(scalerpath)

        cols = xtr.columns
        xtr = pd.DataFrame( scaler.transform(xtr), columns=cols, dtype=np.float32 )
        xvl = pd.DataFrame( scaler.transform(xvl), columns=cols, dtype=np.float32 )
        xte = pd.DataFrame( scaler.transform(xte), columns=cols, dtype=np.float32 )
        # joblib.dump(scaler, outdir/'scaler.pkl')

        # =====================================================
        #      Train baseline (w/o transfer learning)
        # =====================================================
        lg.logger.info('\n{}'.format('-' * 50))
        lg.logger.info('Train LGBM (baseline) ...')
        baseline_lgbm_dir = fold_outdir/'baseline_lgbm'
        os.makedirs(baseline_lgbm_dir)

        # Define and train model
        init_kwargs = {'objective': 'regression', 'n_estimators': 100, 'n_jobs': n_jobs, 'random_state': SEED}
        model = lgb.LGBMModel(**init_kwargs)

        t0 = time()
        fit_kwargs = {'eval_set': (xvl, yvl), 'early_stopping_rounds': 10, 'verbose': False}
        model.fit(xtr, ytr, **fit_kwargs)
        lg.logger.info('Train time: {:.1f} mins'.format( (time()-t0)/60 ))

        # Predict
        pred_ytr = model.predict(xtr).squeeze()
        pred_yvl = model.predict(xvl).squeeze()
        pred_yte = model.predict(xte).squeeze()

        # Append preds
        ptr = mtr.copy(); ptr['fold']=tl_fold; ptr['phase']='tr'; ptr['auc_pred']=pred_ytr
        pvl = mvl.copy(); pvl['fold']=tl_fold; pvl['phase']='vl'; pvl['auc_pred']=pred_yvl
        pte = mte.copy(); pte['fold']=tl_fold; pte['phase']='te'; pte['auc_pred']=pred_yte
        all_bs_lgbm_preds.append( pd.concat([ptr, pvl, pte], axis=0) )

        # Append scores
        lg.logger.info('\nBaseline LGBM:')
        scores = utils.calc_scores(ytr, pred_ytr, yvl, pred_yvl, yte, pred_yte, logger=lg.logger)
        utils.dump_dict(scores, baseline_lgbm_dir/'lgbm_baseline_scores.txt')
        all_bs_lgbm_scores.append(scores) # agg scores
        del model, scores, pred_ytr, pred_yvl, pred_yte, ptr, pvl, pte

        # =====================================================
        #       Load base (pre-trained) model and predict w/o transfer learning
        # =====================================================
        # modelpath = base_model_dir/f'cv{bs_fold}'/'final_model.h5'
        # model = load_model(str(modelpath), custom_objects={'r2_krs': r2_krs})
        modelpath = base_model_dir/f'cv{bs_fold}'/'model_best.h5'
        model = load_model( str(modelpath) )
        plot_model(model, to_file=fold_outdir/'nn_model.png')

        # =====================================================
        #       Transfer learning
        # =====================================================
        # Reset model
        lr_new = 5e-4
        model.compile(loss='mean_squared_error', optimizer=SGD(lr=lr_new, momentum=0.9), metrics=['mae'])

        # Callbacks
        checkpointer = ModelCheckpoint(str(run_outdir/'model_best.h5'), verbose=0, save_weights_only=False, save_best_only=True)
        csv_logger = CSVLogger(fold_outdir/'training.log')

        # Callbacks list
        callbacks = [checkpointer, csv_logger, early_stop, reduce_lr]
        if 'clr' in opt_name: callbacks.append(clr)

        # -------------------------------
        # Choose transfer learning method
        # -------------------------------
        if tl_method=='fe':
            # Feature extractor
            # utils.print_trainable_layers(model, print_all=True)
            utils.pop_layers(model, keep_up_to=keep_up_to)
            # utils.print_trainable_layers(model, print_all=True)

            # Create feature extractor model
            fea_extractor_model = Model(inputs=model.input, outputs=model.layers[-1].output)

            # Extract featues
            xtr_fea_ext = fea_extractor_model.predict(xtr)
            xvl_fea_ext = fea_extractor_model.predict(xvl)
            xte_fea_ext = fea_extractor_model.predict(xte)
            # print(xvl_fea_ext.shape)

            ## Train final model on extracted features
            #lg.logger.info('\n{}'.format('-' * 50))
            #lg.logger.info('Train LGBM feature extractor ...')

            #init_kwargs = {'objective': 'regression', 'n_estimators': 100, 'n_jobs': n_jobs, 'random_state': SEED}
            #model = lgb.LGBMModel(**init_kwargs)

            #t0 = time()
            #fit_kwargs = {'eval_set': (xvl_fea_ext, yvl), 'early_stopping_rounds': 10, 'verbose': False}
            #model.fit(xtr_fea_ext, ytr, **fit_kwargs)
            #lg.logger.info('Train time: {:.1f} mins'.format( (time()-t0)/60 ))

            # Train final model on extracted features
            lg.logger.info('\n{}'.format('-' * 50))
            lg.logger.info('Train RF feature extractor ...')

            init_kwargs = {'n_estimators': 100, 'n_jobs': n_jobs, 'random_state': SEED, 'min_samples_split': 7}
            model = RandomForestRegressor(**init_kwargs)

            t0 = time()
            model.fit(xtr_fea_ext, ytr)
            lg.logger.info('Train time: {:.1f} mins'.format( (time()-t0)/60 ))

            # Predict
            pred_ytr = model.predict(xtr_fea_ext)
            pred_yvl = model.predict(xvl_fea_ext)
            pred_yte = model.predict(xte_fea_ext)

            # Append preds
            ptr = mtr.copy(); ptr['fold']=tl_fold; ptr['phase']='tr'; ptr['auc_pred'] = pred_ytr
            pvl = mvl.copy(); pvl['fold']=tl_fold; pvl['phase']='vl'; pvl['auc_pred'] = pred_yvl
            pte = mte.copy(); pte['fold']=tl_fold; pte['phase']='te'; pte['auc_pred'] = pred_yte
            all_tl_preds.append( pd.concat([ptr, pvl, pte], axis=0) )

            # Append scores
            lg.logger.info('\nScores Feature extractor LGBM:')
            tl_scores = utils.calc_scores(ytr, pred_ytr, yvl, pred_yvl, yte, pred_yte, logger=lg.logger)
            utils.dump_dict(tl_scores, outpath=fold_outdir/'fe_scores.txt')

        elif tl_method=='ft':
            # Finetune
            lg.logger.info('\n{}'.format('-' * 50))
            lg.logger.info('Finetune NN ...')

            # utils.print_trainable_layers(model, print_all=True)
            utils.freeze_layers(model, freeze_up_to=freeze_up_to)
            # utils.print_trainable_layers(model, print_all=True)

            # Fit params
            fit_kwargs = {'epochs': epochs, 'batch_size': batch_size, 'verbose': 1}
            # fit_kwargs['validation_data'] = (xvl, yvl)
            fit_kwargs['validation_split'] = 0.2
            fit_kwargs['callbacks'] = callbacks

            t0 = time()
            history = model.fit(xtr, ytr, **fit_kwargs)

            lg.logger.info('Train time: {:.1f} mins'.format( (time()-t0)/60 ))

            # Predict
            pred_ytr = model.predict(xtr).squeeze()
            pred_yvl = model.predict(xvl).squeeze()
            pred_yte = model.predict(xte).squeeze()

            # Append preds
            ptr = mtr.copy(); ptr['fold']=tl_fold; ptr['phase']='tr'; ptr['auc_pred']=pred_ytr
            pvl = mvl.copy(); pvl['fold']=tl_fold; pvl['phase']='vl'; pvl['auc_pred']=pred_yvl
            pte = mte.copy(); pte['fold']=tl_fold; pte['phase']='te'; pte['auc_pred']=pred_yte
            all_tl_preds.append( pd.concat([ptr, pvl, pte], axis=0) )

            # Append scores (transfer learning)
            lg.logger.info('\nScores using finetune:')
            tl_scores = utils.calc_scores(ytr, pred_ytr, yvl, pred_yvl, yte, pred_yte, logger=lg.logger)
            utils.dump_dict(tl_scores, outpath=fold_outdir/'ft_scores.txt')

            ml_models.plot_prfrm_metrics(history, title=f'Train base model {model_name}',
                                         skp_ep=skp_ep, add_lr=True, outdir=fold_outdir)
            ml_models.save_krs_history(history, fold_outdir)

        del model, pred_ytr, pred_yvl, pred_yte, ptr, pvl, pte
        all_tl_scores.append(tl_scores) # agg scores

    def scores_to_file(scores, outpath, rnd=5):
        """ Dump scores to file. """
        scores = pd.DataFrame.from_dict(scores).T
        scores.columns = ['fold'+str(f+1) for f in range(scores.shape[1])]
        scores.insert(loc=0, column='std', value=scores.std(axis=1))
        scores.insert(loc=0, column='mean', value=scores.iloc[:,1:].mean(axis=1))
        scores.index.name = 'metric'
        scores.reset_index(inplace=True)
        scores.round(rnd).to_csv(outpath, index=False)

    # Dump all scores
    scores_to_file(all_bs_lgbm_scores, run_outdir/'folds_bs_lgbm_scores.csv')
    scores_to_file(all_bs_rf_scores, run_outdir/'folds_bs_rf_scores.csv')
    scores_to_file(all_tl_scores, run_outdir/'folds_tl_scores.csv')

    # Dump all preds
    all_bs_lgbm_preds = pd.concat(all_bs_lgbm_preds, axis=0, ignore_index=True)
    all_bs_lgbm_preds.round(5).to_csv(run_outdir/'all_bs_lgbm_preds.csv', index=False)
    all_bs_rf_preds = pd.concat(all_bs_rf_preds, axis=0, ignore_index=True)
    all_bs_rf_preds.round(5).to_csv(run_outdir/'all_bs_rf_preds.csv', index=False)
    all_tl_preds = pd.concat(all_tl_preds, axis=0, ignore_index=True)
    all_tl_preds.round(5).to_csv(run_outdir/'all_tl_preds.csv', index=False)

    def preds_to_scores(preds, logger=None):
        scores = {}
        for ph in preds['phase'].unique():
            a = preds[ preds['phase']==ph ]
            scores[f'r2_{ph}'] = r2_score(a['auc'], a['auc_pred'])
            scores[f'mae_{ph}'] = mean_absolute_error(a['auc'], a['auc_pred'])
            scores[f'spr_rnk_corr_{ph}'] = spearmanr(a['auc'], a['auc_pred'])[0]

        scores = {k: scores[k] for k in sorted(scores.keys())}
        for k, v in scores.items(): scores[k] = round(v, 4)
        if logger is not None:
            for k, v, in scores.items(): logger.info(f'{k}: {v}')
        return scores

    lg.logger.info('\nLGBM basline scores:')
    lgbm_scores = preds_to_scores(all_bs_lgbm_preds, logger=lg.logger)
    lg.logger.info('\nRF basline scores:')
    rf_scores = preds_to_scores(all_bs_rf_preds, logger=lg.logger)
    lg.logger.info('\nTransfer learning scores:')
    tl_scores = preds_to_scores(all_tl_preds, logger=lg.logger)

    utils.dump_dict(lgbm_scores, run_outdir/'lgbm_baseline_scores.txt')
    utils.dump_dict(rf_scores, run_outdir/'rf_baseline_scores.txt')
    utils.dump_dict(tl_scores, run_outdir/'transfer_learn_scores.txt')

    lg.kill_logger()

def main(args):
    args = parse_args(args)
    args = vars(args)
    ret = run(args)

if __name__ == '__main__':
    """ __name__ == '__main__' explained: www.youtube.com/watch?v=sugvnHA7ElY """
    main(sys.argv[1:])


import numpy as np
from PIL import Image
import pickle
import operator
import matplotlib.pylab as pylab
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE

imgPath = 'E:/dataset/'
trainNum = 7
testNum = 2
crossValidateNum = 1
dataNum = 40
N = 150

train_data = np.empty((dataNum*trainNum, 2679))
train_label = np.empty(dataNum*trainNum)

test_cross_validation_data = np.empty((dataNum*crossValidateNum, 2679))
test_cross_validation_label = np.empty((dataNum*crossValidateNum))

test_data = np.empty((dataNum*testNum, 2679))
test_label = np.empty(dataNum*testNum)

def covertImg():
    imgName = 'olivettifaces.gif'
    # 读取原始图片并转化为np.ndarray，将灰度值由0～256转换到0～1
    img = Image.open(imgPath+imgName)
    img_ndarray = np.asarray(img, dtype='float64')/256

    # 图片大小时1190*942，一共20*20个人脸图，故每张人脸图大小为（1190/20）*（942/20）即57*47=2679
    # 将全部400个样本存储为一个400*2679的数组，每一行即代表一个人脸图，并且第0～9、10～19、20～29...行分别属于同个人脸
    # 另外，用olivettifaces_label表示每一个样本的类别，它是400维的向量，有0～39共40类，代表40个不同的人。
    olivettifaces = np.empty((400, 2679))
    for row in range(20):
        for column in range(20):
            olivettifaces[row * 20 + column] = np.ndarray.flatten(
                img_ndarray[row * 57:(row + 1) * 57, column * 47:(column + 1) * 47])

    # 建一个olivettifaces_label
    olivettifaces_label = np.empty(400)
    for label in range(40):
        olivettifaces_label[label * 10:label * 10 + 10] = label
    olivettifaces_label = olivettifaces_label.astype(np.int)

    # 保存olivettifaces以及olivettifaces_label到olivettifaces.pkl文件
    write_file = open(imgPath+'olivettifaces.pkl', 'wb')
    pickle.dump(olivettifaces, write_file, -1)
    pickle.dump(olivettifaces_label, write_file, -1)
    write_file.close()

def prepareDataSet():
    read_file=open(imgPath+'olivettifaces.pkl','rb')
    faces = pickle.load(read_file)
    label = pickle.load(read_file)
    read_file.close()

    for i in range(40):
        train_data[i * trainNum:i * trainNum + trainNum] = faces[i * 10:i * 10 + trainNum]
        train_label[i * trainNum:i * trainNum + trainNum] = label[i * 10:i * 10 + trainNum]

        test_cross_validation_data[i*crossValidateNum:i*crossValidateNum+crossValidateNum] = faces[i*10+trainNum+testNum : i*10+trainNum+testNum+crossValidateNum]
        test_cross_validation_data[i*crossValidateNum:i*crossValidateNum+crossValidateNum] = label[i*10+trainNum+testNum : i*10+trainNum+testNum+crossValidateNum]

        test_data[i * testNum: i * testNum + testNum] = faces[i * 10 + trainNum : i*10+trainNum+testNum]
        test_label[i * testNum: i * testNum + testNum] = label[i * 10 + trainNum : i*10+trainNum+testNum]

def compute_eigenValues_eigenVectors(arr,label):
    # 计算协方差矩阵的特征值和特征向量，按从大到小顺序排列

    avgArr = np.mean(arr,1)
    avgArr = avgArr.reshape(avgArr.shape[0],1)
    normArr = arr - avgArr
    #计算arr'T * arr

    temp=np.dot(normArr.T, normArr)
    eigenValues,eigenVectors=np.linalg.eig(temp)
    #将数值从大到小排序
    idx=np.argsort(-eigenValues)
    eigenValues=eigenValues[idx]
    #特征向量按列排
    eigenVectors=eigenVectors[:,idx]
    #label排序
    eigenVectorsLabel = label.copy()
    eigenVectorsLabel = eigenVectorsLabel[idx]
    return eigenValues,np.dot(normArr,eigenVectors),eigenVectors, eigenVectorsLabel

def knnOnce(eigenMat,eigenTrainMat,eigenTrainLabel,k):

    dis = np.sum(np.power(eigenMat.reshape(eigenMat.shape[0],1) - eigenTrainMat,2), axis=0)
    sortedDistIndicies = dis.argsort()
    print(dis[sortedDistIndicies[0]])

    classCount = {}

    for i in range(k):
        voteIlabel = eigenTrainLabel[sortedDistIndicies[i]]

        classCount[voteIlabel] = classCount.get(voteIlabel, 0) + 1
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]

def knnModel(trainData, trainLabel, testData, testLabel):
    prepareDataSet()
    num = testData.shape[0]

    eigenTrainValues, eigenTrainMat, eigenTrainVec, eigenTrainLabel = compute_eigenValues_eigenVectors(trainData.T, trainLabel)
    eigenTrainMat = eigenTrainMat[0:N,:]

    eigenValues, eigenMats, eigenVectors, eigenLabel = compute_eigenValues_eigenVectors(testData.T, testLabel)
    errorCount = 0

    for i in range(num):
        result = knnOnce(eigenMats[0:N, i],eigenTrainMat,eigenTrainLabel,5)
        if result != eigenLabel[i]:
            errorCount+=1

    print(str(float(errorCount/num)))

knnModel(train_data, train_label, test_data, test_label)


# -*- coding:utf-8 -*-

import pandas
import matplotlib.pyplot as plt

# Prepare Data
df_raw = pandas.read_csv("../dataset/mpg_ggplot2.csv")
df = df_raw[['cty', 'manufacturer']].groupby('manufacturer').apply(lambda x: x.mean())
df.sort_values('cty', inplace=True)
df.reset_index(inplace=True)

# Draw plot
fig, ax = plt.subplots(figsize=(10, 8), dpi=80)
ax.hlines(
    y=df.index,
    xmin=11,
    xmax=26,
    color='gray',
    alpha=0.7,
    linewidth=1,
    linestyles='dashdot'
)
ax.scatter(
    y=df.index, x=df.cty, s=75, color='#dc2624', alpha=0.7
)

# Title, Label, Ticks and Ylim
ax.set_title('Dot Plot for Highway Mileage', fontdict={'size': 12})

plt.xlabel('Miles Per Gallon', fontsize=12)

ax.set_yticks(df.index)
ax.set_yticklabels(
    df.manufacturer.str.title(),
    fontdict={
        'horizontalalignment': 'right',
        'fontsize': 12
    }
)

plt.xticks(fontsize=12)
ax.set_xlim(10, 27)
plt.savefig("../photos/ranking/dotPlot.png")
plt.show()


from typing import List

"""
Time complexity: O(N)
Space complexity: O(N)
"""

class Solution:
    def longestConsecutive(self, nums):
        longest_streak = 0
        num_set = set(nums)

        for num in num_set:
            if num - 1 not in num_set:
                current_num = num
                current_streak = 1

                while current_num + 1 in num_set:
                    current_num += 1
                    current_streak += 1

                longest_streak = max(longest_streak, current_streak)

        return longest_streak

class Solution:
    def longestConsecutive(self, nums: List[int]) -> int:
        mapping = {num: {"visited": False, "acc": 1} for num in nums}
        longest = 0
        for num in nums:
            if mapping[num]['visited']:
                continue
            acc = 0
            originalNum = num
            while num in mapping:
                acc += mapping[num]["acc"]
                if mapping[num]["visited"]:
                    break
                mapping[num]["visited"] = True
                num -= 1

            mapping[originalNum]["acc"] = acc
            if acc > longest:
                longest = acc
        return longest

s = Solution()
# nums = [0, 3, 7, 2, 5, 8, 4, 6, 0, 1]
nums = [100, 4, 200, 1, 3, 2, 5, 30, 40, 8, 7, 6]
result = s.longestConsecutive(nums)
print(result)

"""

"""


from typing import List, Tuple

import numpy as np

import croupier
import dice
from dice_type import DiceType, AVAILABLE_DICES

def calculate_viterbi_probabilities(observations: np.ndarray, fair_dice: dice.Dice, loaded_dice: dice.Dice,
                                    initial_dice_probability: np.ndarray, transition_matrix: np.ndarray) -> np.ndarray:
    observations_rows = observations.shape[0]

    # Initial probabilities of each dices depends on random choice of croupier and probability of an observation
    viterbi_probabilities = np.zeros([observations_rows, AVAILABLE_DICES])
    viterbi_probabilities[0] = np.array([
        initial_dice_probability[DiceType.FAIR] * fair_dice.probabilities[observations[0]],
        initial_dice_probability[DiceType.LOADED] * loaded_dice.probabilities[observations[0]],
    ])

    # Iteratively compute dice probabilities based on previous observations and probabilities
    for t, observation in enumerate(observations[1:], 1):
        viterbi_probabilities[t] = np.array([
            fair_dice.probabilities[observation] * np.max([
                # Previously we've used fair dice and now we use fair dice
                viterbi_probabilities[t-1][DiceType.FAIR] * transition_matrix[DiceType.FAIR, DiceType.FAIR],
                # Previously we've used loaded dice and now we use fair dice
                viterbi_probabilities[t-1][DiceType.LOADED] * transition_matrix[DiceType.LOADED, DiceType.FAIR],
            ]),
            loaded_dice.probabilities[observation] * np.max([
                # Previously we've used fair dice and now we use loaded dice
                viterbi_probabilities[t-1][DiceType.FAIR] * transition_matrix[DiceType.FAIR, DiceType.LOADED],
                # Previously we've used loaded dice and now we use loaded dice
                viterbi_probabilities[t-1][DiceType.LOADED] * transition_matrix[DiceType.LOADED, DiceType.LOADED],
            ]),
        ])

        # Normalize current probabilities, so that values won't be too low over time
        viterbi_probabilities[t] /= np.sum(viterbi_probabilities[t])

    return viterbi_probabilities

def calculate_forward_probabilities(observations: np.ndarray, fair_dice: dice.Dice, loaded_dice: dice.Dice,
                                    initial_dice_probability: np.ndarray, transition_matrix: np.ndarray) -> np.ndarray:
    observations_rows = observations.shape[0]

    # Initial probabilities for alpha equals probability of random choice of a given dice and roll of given observation
    alpha_probabilities = np.zeros([observations_rows, AVAILABLE_DICES])
    alpha_probabilities[0] = np.array([
        initial_dice_probability[DiceType.FAIR] * fair_dice.probabilities[observations[0]],
        initial_dice_probability[DiceType.LOADED] * loaded_dice.probabilities[observations[0]],
    ])

    # Iteratively compute probabilities based on previous ones
    for t, observation in enumerate(observations[1:], 1):
        alpha_probabilities[t] = np.array([
            fair_dice.probabilities[observation] * np.sum([
                # Previously we've used fair dice and now we use fair dice
                alpha_probabilities[t-1][DiceType.FAIR] * transition_matrix[DiceType.FAIR, DiceType.FAIR],
                # Previously we've used loaded dice and now we use fair dice
                alpha_probabilities[t-1][DiceType.LOADED] * transition_matrix[DiceType.LOADED, DiceType.FAIR]
            ]),
            loaded_dice.probabilities[observation] * np.sum([
                # Previously we've used fair dice and now we use loaded dice
                alpha_probabilities[t-1][DiceType.FAIR] * transition_matrix[DiceType.FAIR, DiceType.LOADED],
                # Previously we've used loaded dice and now we use loaded dice
                alpha_probabilities[t-1][DiceType.LOADED] * transition_matrix[DiceType.LOADED, DiceType.LOADED]
            ]),
        ])

        # Normalize current probabilities, so that values won't be too low over time
        alpha_probabilities[t] /= np.sum(alpha_probabilities[t])

    return alpha_probabilities

def calculate_backward_probabilities(observations: np.ndarray, fair_dice: dice.Dice, loaded_dice: dice.Dice,
                                     initial_dice_probability: np.ndarray, transition_matrix: np.ndarray) -> np.ndarray:
    observations_rows = observations.shape[0]

    # Initial probabilities for beta equals to 1 for each dice. Keep in mind that we need to iterate from the back!
    beta_probabilities = np.zeros([observations_rows, AVAILABLE_DICES])
    beta_probabilities[observations_rows-1] = np.array([1, 1])

    for t, observation in reversed(list(enumerate(observations))[1:]):
        beta_probabilities[t-1] = np.array([
            np.sum([
                # In the observation T croupier used fair dice and fair dice in T-1
                transition_matrix[DiceType.FAIR, DiceType.FAIR] * fair_dice.probabilities[observation]
                  * beta_probabilities[t][DiceType.FAIR],
                # In the observation T croupier used loaded dice and fair dice in T-1
                transition_matrix[DiceType.FAIR, DiceType.LOADED] * loaded_dice.probabilities[observation]
                  * beta_probabilities[t][DiceType.LOADED],
            ]),
            np.sum([
                # In the observation T croupier used fair dice and loaded dice in T-1
                transition_matrix[DiceType.LOADED, DiceType.FAIR] * fair_dice.probabilities[observation]
                  * beta_probabilities[t][DiceType.FAIR],
                # In the observation T croupier used loaded dice and loaded dice in T-1
                transition_matrix[DiceType.LOADED, DiceType.LOADED] * loaded_dice.probabilities[observation]
                  * beta_probabilities[t][DiceType.LOADED],
            ]),
        ])

        # Again, as previously, normalize all probabilities, so that values won't be too low over time
        beta_probabilities[t-1] /= np.sum(beta_probabilities[t-1])

    return beta_probabilities

def baum_welch(multiple_observations: List[np.ndarray], epochs: int = 50) -> Tuple[dice.Dice, dice.Dice, np.ndarray, np.ndarray]:
    # Initialize first dice parameters
    first_dice_probabilities = np.random.rand(6)
    first_dice_probabilities /= first_dice_probabilities.sum()  # Make probabilities sum up to 1.0
    first_dice = dice.Dice(first_dice_probabilities)

    # Initialize second dice parameters
    second_dice_probabilities = np.random.rand(6)
    second_dice_probabilities /= second_dice_probabilities.sum()  # Make probabilities sum up to 1.0
    second_dice = dice.Dice(second_dice_probabilities)

    # Initialize probabilities which were used to randomize first dice
    initial_dice_probability = np.random.rand(2)
    initial_dice_probability /= initial_dice_probability.sum()  # Make probabilities sum up to 1.0

    # Initialize transition probabilities
    transition_matrix = np.random.rand(2, 2)
    transition_matrix /= np.sum(transition_matrix, axis=1)[:, None]  # Make probabilities sum up to 1.0 for each dice

    # Train our Hidden Markov Model in epochs
    for epoch in range(epochs):
        # Prepare placeholders for collecting computations from all observations
        new_transition_matrix = np.zeros((2, 2))
        new_initial_dice_probability = np.zeros(2)
        new_first_dice_probabilities = np.zeros(6)
        new_second_dice_probabilities = np.zeros(6)
        fitness = 0

        # In each epoch walk through all collected observations
        for observations in multiple_observations:
            observations_rows = observations.shape[0]

            # E Step - use forward and backward pass to calculate alphas and betas
            alpha_probabilities = calculate_forward_probabilities(observations, first_dice, second_dice,
                                                                  initial_dice_probability, transition_matrix)
            beta_probabilities = calculate_backward_probabilities(observations, first_dice, second_dice,
                                                                  initial_dice_probability, transition_matrix)
            aposteriori_probabilities = np.multiply(alpha_probabilities, beta_probabilities)

            # Collect probabilities for each observations which can be defined as P(O|parameters)
            probability_of_observation = initial_dice_probability[0] * beta_probabilities[0][0] \
                                         + initial_dice_probability[1] * beta_probabilities[0][1]
            fitness += probability_of_observation

            # KSI (Greek Letter) parameters are known as probability of being in state `i` at time `t` and in
            # state `j` at time `t+1`, given the model and the sequence of observations
            ksi = np.zeros([observations_rows, 2, 2])
            for t in range(observations_rows-1):
                denominator = (alpha_probabilities[t][0] * beta_probabilities[t][0] + alpha_probabilities[t][1] * beta_probabilities[t][1])
                ksi[t, 0, 0] = (alpha_probabilities[t][0] * transition_matrix[0, 0]
                                * first_dice_probabilities[observations[t+1]] * beta_probabilities[t+1][0]) / denominator
                ksi[t, 0, 1] = (alpha_probabilities[t][0] * transition_matrix[0, 1]
                                * second_dice_probabilities[observations[t+1]] * beta_probabilities[t+1][1]) / denominator
                ksi[t, 1, 0] = (alpha_probabilities[t][1] * transition_matrix[1, 0]
                                * first_dice_probabilities[observations[t+1]] * beta_probabilities[t+1][0]) / denominator
                ksi[t, 1, 1] = (alpha_probabilities[t][1] * transition_matrix[1, 1]
                                * second_dice_probabilities[observations[t+1]] * beta_probabilities[t+1][1]) / denominator

            # Gamma (another Greek Letter) parameters stands for probability of being in state `i` at time `t`
            gamma = np.zeros([observations_rows, 2])
            for t in range(observations_rows-1):
                gamma[t, 0] = np.sum(ksi[t, 0, :])
                gamma[t, 1] = np.sum(ksi[t, 1, :])

            # Compute initial probabilities for this observation and add it for later use
            new_initial_dice_probability += gamma[0] / np.sum(gamma[0])

            # Compute transition matrix and add it up to for later use
            new_transition_matrix[0, 0] += np.sum(ksi[:, 0, 0]) / np.sum(gamma[:, 0]) / probability_of_observation
            new_transition_matrix[0, 1] += np.sum(ksi[:, 0, 1]) / np.sum(gamma[:, 0]) / probability_of_observation
            new_transition_matrix[1, 0] += np.sum(ksi[:, 1, 0]) / np.sum(gamma[:, 1]) / probability_of_observation
            new_transition_matrix[1, 1] += np.sum(ksi[:, 1, 1]) / np.sum(gamma[:, 1]) / probability_of_observation

            # Compute probabilities for each dice (hidden states) and collect them for later use
            for wall in range(6):
                # Emission probabilities can be computed as expected number of times in state `j` and observing `k-th` wall on dice
                # divided by expected number of times in state `j`
                new_first_dice_probabilities[wall] = np.sum(np.take(aposteriori_probabilities[:, 0], np.where(observations == wall)))
                new_first_dice_probabilities[wall] /= np.sum(aposteriori_probabilities[:, 0]) / probability_of_observation
                new_second_dice_probabilities[wall] = np.sum(np.take(aposteriori_probabilities[:, 1], np.where(observations == wall)))
                new_second_dice_probabilities[wall] /= np.sum(aposteriori_probabilities[:, 1]) / probability_of_observation

        # Normalize all values for probabilities, so that they won't blow up quickly...
        initial_dice_probability = new_initial_dice_probability / np.sum(new_initial_dice_probability)
        transition_matrix = new_transition_matrix / np.sum(new_transition_matrix, axis=1)[:, None]
        first_dice_probabilities = new_first_dice_probabilities / np.sum(new_first_dice_probabilities)
        second_dice_probabilities = new_second_dice_probabilities / np.sum(new_second_dice_probabilities)

        # Put dice probabilities into Dice instances
        first_dice = dice.Dice(first_dice_probabilities)
        second_dice = dice.Dice(second_dice_probabilities)

        # Some logging is always good :)
        print('= EPOCH #{} ='.format(epoch))
        print('Transition Matrix:', transition_matrix)
        print('Initial Dice Probability:', initial_dice_probability)
        print('First Dice:', first_dice_probabilities)
        print('Second Dice:', second_dice_probabilities)
        print('Fitness:', fitness)
        print()

    return first_dice, second_dice, initial_dice_probability, transition_matrix


# -*- coding: utf-8 -*-
"""
zfec_segmenter.py

Encodes/decodes segments using zfec.
"""
from zfec.easyfec import Encoder, Decoder

from tools.data_definitions import block_size, encoded_block_slice_size

def _slice_count(data_size, slice_size):
    if data_size == 0:
        return 0
    if data_size % slice_size == 0:
        return data_size / slice_size
    return (data_size / slice_size) + 1

def _slice_generator(data, slice_size):
    start_pos = 0
    end_pos = slice_size
    while start_pos < len(data):
        yield data[start_pos:end_pos]
        start_pos = end_pos
        end_pos += slice_size

class ZfecSegmenter(object):
    def __init__(self, min_segments, num_segments):
        self.min_segments = min_segments
        self.num_segments = num_segments

    def padding_size(self, data):
        modulus = len(data) % self.min_segments
        return (0 if modulus == 0 else self.min_segments - modulus)

    def encode(self, data):
        """
        data
            data to be encoded

        return
            a list of num_segments of encoded zfec shares concatenated
        """
        return_list = [list() for _ in range(self.num_segments)]
        encoder = Encoder(self.min_segments, self.num_segments)
        for data_slice in _slice_generator(data, block_size):
            for segment_list, zfec_share in  zip(
                return_list, encoder.encode(data_slice)
            ):
                segment_list.append(zfec_share)

        return ["".join(sublist) for sublist in return_list]

    def decode(self, segments, segment_numbers, padding_size):
        """
        segments
            a list of num_segments strings each containing concatenated
            encoded blocks
        segment_numbers
            a list of ints giving the segment nimbers of the segments (1..n)
        padding_size
            the zfec padding size of the last block (earlier blocks are 0)
        """
        data_list = list()
        slice_count = _slice_count(len(segments[0]), encoded_block_slice_size)
        slicegens = [
            _slice_generator(s, encoded_block_slice_size) for s in segments
        ]
        decoder = Decoder(self.min_segments, self.num_segments)
        zfec_segment_numbers = [n-1 for n in segment_numbers]

        # accumulate all but the last slice with padding set to 0
        for _ in range(slice_count-1):
            encoded_blocks = [slicegen.next() for slicegen in slicegens]
            data_list.append(
                decoder.decode(encoded_blocks, zfec_segment_numbers, 0)
            )

        # accumulate the last slice using the padding size
        encoded_blocks = [slicegen.next() for slicegen in slicegens]
        data_list.append(
            decoder.decode(encoded_blocks, zfec_segment_numbers, padding_size)
        )

        return "".join(data_list)


from datetime import datetime, timedelta
import dateutil.parser
from pytz import timezone
import re
from math import floor, ceil, radians, sin, cos, atan2, sqrt

def haversine(lat1, lon1, lat2, lon2):
    R = 6367444.7
    phi1 = radians(lat1)
    phi2 = radians(lat2)
    delta_phi = radians(lat2 - lat1)
    delta_lambda = radians(lon2 - lon1)
    sin_delta_p = sin(delta_phi / 2.)
    sin_delta_l = sin(delta_lambda / 2.)
    a = sin_delta_p * sin_delta_p + cos(phi1) * cos(phi2) * sin_delta_l * sin_delta_l
    c = 2. * atan2(sqrt(a), sqrt(1. - a));
    return R * c;

def camel_case_to_py(txt):
    retval = ''
    for c in txt:
        if c.isupper():
            retval += '_%s' % c.lower()
        else:
            retval += c
    return retval

def attributes_to_fields(fields, obj, data, remap = None):
    for field in fields:
        k = camel_case_to_py(field)
        if remap is None:
            setattr(obj, k, data[field])
        else:
            setattr(obj, k, remap(data[field]))

def parse_timedelta(txt):
    RGX = re.compile('^((?P<hours>\d+):)?((?P<minutes>\d+):)?(?P<seconds>\d+(\.\d+)?)')
    match = RGX.match(txt)
    if not match:
        return None
    parms = {'hours': 0, 'minutes': 0, 'seconds': 0.0}
    for key, val in match.groupdict().items():
        if val and key in parms:
            parms[key] = float(val)
    return timedelta(**parms)

class ActivityTags(object):
    TERRAIN_TRAIL = 'TRAIL'
    EMOTION_UNSTOPPABLE = 'UNSTOPPABLE'
    WEATHER_PARTLY_SUNNY = 'PARTLY_SUNNY'

    def load(self, json_data):
        d = {}
        for item in json_data:
            d[item['tagType'].lower()] = item['tagValue']

        attributes_to_fields(['terrain', 'emotion', 'weather'], self, d)
        attributes_to_fields(['temperature'], self, d, int)

    def __init__(self):
        super(ActivityTags, self).__init__()
        self.temperature = None
        self.terrain = None
        self.emotion = None
        self.weather = None

class ActivityMetricsSummary(object):
    def load(self, json_data):
        attributes_to_fields(['calories', 'fuel', 'steps'], self, json_data, int)
        attributes_to_fields(['distance'], self, json_data, float)
        attributes_to_fields(['duration'], self, json_data, parse_timedelta)

    def __init__(self):
        super(ActivityMetricsSummary, self).__init__()
        self.calories = None
        self.fuel = None
        self.distance = None
        self.steps = None
        self.duration = None
        self.elevation_loss = None
        self.elevation_gain = None
        self.elevation_max = None
        self.elevation_min = None

class ActivityMetric(object):
    # Other metric types: https://developer.nike.com/documentation/api-docs/reference/interval-and-summary-metric-types.html
    METRIC_GPS = '__GPS__' # Not defined by Nike+
    METRIC_TIME = '__TIME__' # Not defined by Nike+
    METRIC_DISTANCE = 'DISTANCE'
    METRIC_SPEED = 'SPEED'
    METRIC_GPSSIGNALSTRENGTH = 'GPSSIGNALSTRENGTH'
    METRIC_HEARTRATE = 'HEARTRATE'
    METRIC_FUEL = 'FUEL'
    UNIT_SEC = 'SEC'
    UNIT_MIN = 'MIN'

    @classmethod
    def interpolate(cls, alpha, zero, one):
        return (1. - alpha) * zero + alpha * one

    def load(self, json_data):
        attributes_to_fields(['intervalUnit', 'metricType'], self, json_data)
        self.interval = int(json_data['intervalMetric'])
        if self.interval_unit == ActivityMetric.UNIT_MIN:
            self.interval *= 60
            self.interval_unit = ActivityMetric.UNIT_SEC

        self.values = list(map(float, json_data['values']))

    def sample(self, idx):
        if idx < 0:
            return self.values[0]
        elif idx >= len(self.values) - 1:
            return self.values[-1]
        elif float(int(idx)) == idx:
            return self.values[int(idx)]
        else:
            val_prev = self.values[int(floor(idx))]
            val_next = self.values[int(ceil(idx))]
            alpha = idx - floor(idx)
            return self.__class__.interpolate(alpha, val_prev, val_next)

    def resample_uniform(self, new_interval):
        unit = new_interval / self.interval
        new_values = []
        x = 0.
        while x <= len(self.values) - 1:
            new_values.append(self.sample(x))
            x += unit
        self.values = new_values
        self.interval = new_interval

    def resample(self, idxs, new_interval_secs):
        new_values = []
        for i in idxs:
            new_values.append(self.sample(i))
        self.values = new_values
        self.interval = new_interval_secs

    def __init__(self):
        super(ActivityMetric, self).__init__()
        self.metric_type = None
        self.interval_unit = None
        self.interval = None
        self.values = None

class ActivityMetricGPS(ActivityMetric):
    @classmethod
    def interpolate(cls, alpha, zero, one):
        return {
            'latitude': (1. - alpha) * zero['latitude'] + alpha * one['latitude'],
            'longitude': (1. - alpha) * zero['longitude'] + alpha * one['longitude'],
            'elevation': (1. - alpha) * zero['elevation'] + alpha * one['elevation']
        }

    def compute_distance(self, max_dist):
        self.values[0]['distance'] = 0.0
        prev_lat = self.values[0]['latitude']
        prev_lon = self.values[0]['longitude']
        prev_dist = 0.0
        for i in range(1, len(self.values)):
            lat = self.values[i]['latitude']
            lon = self.values[i]['longitude']
            prev_dist += haversine(prev_lat, prev_lon, lat, lon) / 1000.
            self.values[i]['distance'] = prev_dist
            prev_lat, prev_lon = lat, lon
        last_dist = self.values[-1]['distance']
        scale_factor = max_dist / last_dist
        for i in range(0, len(self.values)):
            self.values[i]['distance'] *= scale_factor
        return last_dist - max_dist

    def map_dist_to_indices(self, metric_dist):
        retval = []
        mdist_i = 0
        prev_mdist = 0.
        n_mdist = len(metric_dist.values)
        for item in self.values:
            dist = item['distance']
            while mdist_i < n_mdist and metric_dist.values[mdist_i] < dist:
                prev_mdist = metric_dist.values[mdist_i]
                mdist_i += 1
            if mdist_i >= n_mdist:
                retval.append(n_mdist - 1)
            if metric_dist.values[mdist_i] == dist:
                retval.append(mdist_i)
                continue
            # interpolate
            assert(mdist_i > 0 and prev_mdist == metric_dist.values[mdist_i - 1])
            assert(dist >= prev_mdist)
            alpha = (dist - prev_mdist) / (metric_dist.values[mdist_i] - prev_mdist)
            retval.append(float(mdist_i) - 1. + alpha)
            assert(abs(metric_dist.sample(retval[-1]) - dist) < 0.001)
        return retval

    def load(self, json_data):
        attributes_to_fields(['elevationLoss', 'elevationGain', 'elevationMax', 'elevationMin'], self, json_data, float)
        attributes_to_fields(['intervalUnit'], self, json_data)
        self.interval = int(json_data['intervalMetric'])
        self.values = json_data['waypoints']

    def __init__(self):
        super(ActivityMetricGPS, self).__init__()
        self.elevation_loss = None
        self.elevation_gain = None
        self.elevation_min = None
        self.elevation_max = None

class ActivityMetrics(object):
    def load(self, json_data):
        for json_metric in json_data:
            metric = ActivityMetric()
            metric.load(json_metric)
            self.all_metrics[metric.metric_type] = metric

    def synchronize_to_distance(self):
        assert(self.distance is not None)
        target_len = len(self.distance.values)
        for k in self.all_metrics:
            if k == ActivityMetric.METRIC_GPS or k == ActivityMetric.METRIC_DISTANCE:
                continue
            metric = self.all_metrics[k]
            metric.resample_uniform(float(self.distance.interval))
            if len(metric.values) > target_len:
                del metric.values[target_len:]
            else:
                metric.values += [metric.values[-1]] * (target_len - len(metric.values))

    def synchronize_to_gps(self):
        assert(self.gps is not None and self.distance is not None)
        self.gps.compute_distance(self.distance.values[-1])
        idxs = self.gps.map_dist_to_indices(self.distance)
        assert(len(idxs) == len(self.gps.values))
        for k in self.all_metrics:
            if k == ActivityMetric.METRIC_GPS or k == ActivityMetric.METRIC_DISTANCE:
                continue
            metric = self.all_metrics[k]
            if len(metric.values) != len(self.distance.values):
                metric.resample_uniform(float(self.distance.interval))
            metric.resample(idxs, self.gps.interval)
        self.distance.resample(idxs, self.gps.interval)

    def pack(self, metrics = None):
        if metrics is None:
            metrics = self.all_metrics.keys()

        # Make sure all the metrics are in sync
        any_metric = self.gps if self.gps is not None else self.all_metrics[metrics[0]]
        assert(any_metric.interval_unit == ActivityMetric.UNIT_SEC)
        for k in metrics:
            assert(len(self.all_metrics[k].values) == len(any_metric.values))
            assert(self.all_metrics[k].interval == any_metric.interval)
            assert(self.all_metrics[k].interval_unit == any_metric.interval_unit)

        packed_values = []
        for i in range(0, len(any_metric.values)):
            packed_values.append({})
            for k in metrics:
                packed_values[-1][k] = self.all_metrics[k].values[i]
                packed_values[-1][ActivityMetric.METRIC_TIME] = timedelta(
                    seconds=float(any_metric.interval) * float(i))
        return packed_values

    def __getattribute__(self, key):
        if key == 'distance':
            return self.all_metrics.get(ActivityMetric.METRIC_DISTANCE)
        elif key == 'speed':
            return self.all_metrics.get(ActivityMetric.METRIC_SPEED)
        elif key == 'gps_signal_strength':
            return self.all_metrics.get(ActivityMetric.METRIC_GPSSIGNALSTRENGTH)
        elif key == 'heart_rate':
            return self.all_metrics.get(ActivityMetric.METRIC_HEARTRATE)
        elif key == 'fuel':
            return self.all_metrics.get(ActivityMetric.METRIC_FUEL)
        elif key == 'gps':
            return self.all_metrics.get(ActivityMetric.METRIC_GPS)
        else:
            return super(ActivityMetrics, self).__getattribute__(key)

    def __init__(self):
        super(ActivityMetrics, self).__init__()
        self.all_metrics = {}

class ActivityData(object):
    # The remaining activity types: https://developer.nike.com/documentation/api-docs/reference/activity-types.html
    ACTIVITY_TYPE_RUN = 'RUN'
    STATUS_COMPLETE = 'COMPLETE'
    # The remaining device types: https://developer.nike.com/documentation/api-docs/reference/device-types.html
    DEVICE_TYPE_IPHONE = 'IPHONE'

    def load(self, json_data, gps_data = None):
        # Import trivial attributes
        attributes_to_fields(['activityId', 'activityType', 'status', 'deviceType'], self, json_data)
        attributes_to_fields(['isGpsActivity'], self, json_data, bool)
        self.metrics_summary.load(json_data['metricSummary'])
        self.tags.load(json_data['tags'])
        self.metrics.load(json_data['metrics'])
        # Cast time with the correct timezone
        tz = timezone(json_data['activityTimeZone'])
        self.start_time = tz.localize(dateutil.parser.parse(json_data['startTime'], ignoretz=True))
        if gps_data is not None:
            self.load_gps_data(gps_data)

    def load_gps_data(self, gps_data):
        gps = ActivityMetricGPS()
        gps.load(gps_data)
        self.metrics.all_metrics[ActivityMetric.METRIC_GPS] = gps
        self.metrics_summary.elevation_max = gps.elevation_max
        self.metrics_summary.elevation_min = gps.elevation_min
        self.metrics_summary.elevation_gain = gps.elevation_gain
        self.metrics_summary.elevation_loss = gps.elevation_loss

    def __init__(self):
        super(ActivityData, self).__init__()
        self.activity_id = None
        self.activity_type = None
        self.start_time = None
        self.status = None
        self.device_type = None
        self.is_gps_activity = None
        self.metrics_summary = ActivityMetricsSummary()
        self.metrics = ActivityMetrics()
        self.tags = ActivityTags()

if __name__ == '__main__':
    import simplejson
    import xml.dom.minidom as DOM

    with open('/Users/Spak/Desktop/last.json', 'r') as fs:
        gpsdata = simplejson.load(fs)
    with open('/Users/Spak/Desktop/last2.json', 'r') as fs:
        actdata = simplejson.load(fs)
    a = ActivityData()
    a.load(actdata, gpsdata)
    # diff = a.metrics.gps.compute_distance(a.metrics_summary.distance)
    # print('Difference between estimated length and computed length: %f m' % diff)
    # a.metrics.synchronize_to_distance()
    # # TEST
    # for k in a.metrics.all_metrics:
    #     if k == ActivityMetric.METRIC_GPS:
    #         continue
    #     assert(len(a.metrics.all_metrics[k].values) == len(a.metrics.distance.values))
    # # ----
    # idxs = a.metrics.gps.map_dist_to_indices(a.metrics.distance)
    # # TEST
    # for i, j in enumerate(idxs):
    #     assert(abs(a.metrics.distance.sample(j) - a.metrics.gps.values[i]['distance']) < 0.1)
    # # ----
    a.metrics.synchronize_to_gps()
    trackpoints = a.metrics.pack()

    # Convert to xml
    doc = DOM.Document()
    xml_trkseg = doc.createElement('trkseg')
    for tp in trackpoints:
        xml_tp = doc.createElement('trkpt')
        xml_tp.setAttribute('lat', str(tp[ActivityMetric.METRIC_GPS]['latitude']))
        xml_tp.setAttribute('lon', str(tp[ActivityMetric.METRIC_GPS]['longitude']))
        ele_tp = doc.createElement('ele')
        ele_tp.appendChild(doc.createTextNode(str(tp[ActivityMetric.METRIC_GPS]['elevation'])))
        xml_tp.appendChild(ele_tp)
        time_tp = doc.createElement('time')
        real_time = a.start_time + tp[ActivityMetric.METRIC_TIME]
        time_tp.appendChild(doc.createTextNode(real_time.isoformat()))
        xml_tp.appendChild(time_tp)
        ext_tp = doc.createElement('extensions')
        xml_tp.appendChild(ext_tp)
        heart_tp = doc.createElement('hr')
        heart_tp.appendChild(doc.createTextNode(str(int(tp[ActivityMetric.METRIC_HEARTRATE]))))
        ext_tp.appendChild(heart_tp)
        xml_trkseg.appendChild(xml_tp)
    xml_trk = doc.createElement('trk')
    xml_trk.appendChild(xml_trkseg)
    xml_gpx = doc.createElement('gpx')
    xml_gpx.setAttribute('version', '1.1')
    xml_gpx.setAttribute('creator', 'PyGPX')
    xml_gpx.appendChild(xml_trk)
    print("<?xml version=\"1.0\" ?>")
    print(xml_gpx.toprettyxml())


"""GraphML importer/exporter."""
from __future__ import annotations

import json
from typing import TYPE_CHECKING, Any, Type
from xml.etree.ElementTree import Element

import attrs
from networkx import Graph
from networkx.readwrite.graphml import GraphMLReader, GraphMLWriter

from .. import representation as rep

if TYPE_CHECKING:
    from networkx.readwrite.graphml import GraphMLKeys

    WriterBase = GraphMLWriter[rep.Node, Any, rep.Edge]
    ReaderBase = GraphMLReader[rep.Graph]
else:
    WriterBase = GraphMLWriter
    ReaderBase = GraphMLReader

class CustomGraphMLWriter(WriterBase):
    def _add_edge_attributes(
        self, scope: str, xml_obj: Element, edge: rep.Edge, default: dict[str, object]
    ) -> None:
        self.attribute_types[("type", scope)].add(str)
        self.attributes[xml_obj].append(
            ("type", edge.__class__.__name__.upper(), scope, default.get("type"))
        )
        if attrs.has(type(edge)):
            for k, v in attrs.asdict(edge).items():
                self.attribute_types[(k, scope)].add(str)
                self.attributes[xml_obj].append(
                    (k, json.dumps(v), scope, default.get(k))
                )

    def add_nodes(
        self, g: Graph[rep.Node, Any, rep.Edge], graph_element: Element
    ) -> None:
        for node, _ in g.nodes(data=True):
            node_element = self.myElement("node", id=str(node.node_id))
            self.add_attributes(
                "node", node_element, self._dump_node_attributes(node), {}
            )
            graph_element.append(node_element)

    def _dump_node_attributes(self, node: rep.Node) -> dict[str, str]:
        result = {
            "node_type": node.__class__.__name__,
        }
        for k, v in attrs.asdict(node).items():
            if v is None:
                continue
            result[k] = json.dumps(v)
        return result

    def add_edges(
        self, g: Graph[rep.Node, Any, rep.Edge], graph_element: Element
    ) -> None:
        assert isinstance(g, rep.Graph)

        for u, v, key, data in g.edges(data=True, keys=True):
            edge_element = self.myElement(
                "edge",
                source=str(u.node_id),
                target=str(v.node_id),
                id=f"{u.node_id}-{v.node_id}-{key}",
            )
            self._add_edge_attributes("edge", edge_element, data["type"], {})
            graph_element.append(edge_element)

class CustomGraphMLReader(ReaderBase):
    def __init__(
        self,
        node_type: Type[rep.Node] = rep.Node,
        edge_key_type: Type[int] = int,
        force_multigraph: bool = False,
    ) -> None:
        super().__init__(node_type, edge_key_type, force_multigraph)
        self._node_map: dict[int, rep.Node] = {}
        self.multigraph = True

    def make_graph(
        self,
        graph_xml: Element,
        graphml_keys: dict[str, GraphMLKeys],
        defaults: object,
        G: rep.Graph | None = None,
    ) -> rep.Graph:
        data = self.decode_data_elements(graphml_keys, graph_xml)
        return super().make_graph(graph_xml, graphml_keys, defaults, rep.Graph(**data))

    def add_node(
        self,
        g: rep.Graph,
        node_xml: Element,
        graphml_keys: dict[str, GraphMLKeys],
        defaults: object,
    ) -> None:
        # get data/attributes for node
        data = self.decode_data_elements(graphml_keys, node_xml)
        node = self._load_node(data)

        self._node_map[node.node_id] = node
        g.add_node(node)

    def _load_node(self, data: dict[str, str]) -> rep.Node:
        node_type = data["node_type"]
        node_cls: Type[rep.Node] = getattr(rep, node_type)

        node_attrs = {
            k: json.loads(v)
            for k, v in data.items()
            if k not in ("node_type", "node_id")
        }
        if "location" in node_attrs:
            node_attrs["location"] = rep.NodeLocation(**node_attrs["location"])

        node = node_cls(**node_attrs)
        node.node_id = json.loads(data["node_id"])
        return node

    def add_edge(
        self, g: rep.Graph, edge_element: Element, graphml_keys: dict[str, GraphMLKeys]
    ) -> None:
        source = self._node_map[int(edge_element.get("source"))]  # type: ignore[arg-type]
        target = self._node_map[int(edge_element.get("target"))]  # type: ignore[arg-type]
        data = self.decode_data_elements(graphml_keys, edge_element)

        edge_type = data["type"]

        edge_cls = getattr(rep, edge_type_to_name.get(edge_type, edge_type.title()))
        edge = edge_cls(**{k: json.loads(v) for k, v in data.items() if k != "type"})

        g.add_edge(source, target, edge)

edge_type_to_name = {
    "DEFLOOPITEM": "DefLoopItem",
    "DEFINEDIF": "DefinedIf",
}

def dump_graph(g: rep.Graph) -> str:
    if not g:
        return ""

    writer = CustomGraphMLWriter(prettyprint=True)
    writer.add_graph_element(g)
    return str(writer)

def import_graph(graphml_str: str, role_id: str, role_version: str) -> rep.Graph:
    if not graphml_str:
        return rep.Graph(role_id, role_version)

    reader = CustomGraphMLReader()

    graphs = list(reader(string=graphml_str))
    assert len(graphs) == 1

    return graphs[0]