# TODO: License

"""
.. module:: config
"""

from __future__ import print_function
from os import path, makedirs
import numpy as np

from .dataset import dataset
from .paircounts import paircounts
from .combine import combine
from .dndz import dndz
from .plot import plot
from .compare import compare
from .compare_noz_fom import compare_noz_fom
from .FoM import FoM
from .plot_FoM import plot_FoM

def read_config(file_name):
    """Read a configuration dictionary from a file

    :param file_name:   yaml file name which we read
    """
    import yaml
    with open(file_name) as f_in:
        config = yaml.load(f_in.read())
    return config

def save_config(config, file_name):
    """Take a configuration dictionary and save to file

    :param config:  Dictionary of configuration
    :param file_name: file we are saving to
    """
    import yaml
    with open(file_name, 'w') as f:
        f.write(yaml.dump(config, default_flow_style=False))

def make_directories(config):
    # go through the known dataset paths and make directories
    paths = {'dataset': ['dataset_path'],
             'paircounts': ['paircounts_path'],
             'combine': ['combine_path'],
             'dndz': ['dndz_path'],
             'plot': ['plot_path'],
             'compare': ['compare_path'],
             'compare_noz_fom': ['compare_noz_fom_path'],
              'FoM' : ['FoM_path'],
              'plot_FoM' : ['plot_path'],}
    for i in paths:
        if i in config:
            for j in paths[i]:
                if j in config[i]:
                    check_make(path.dirname(config[i][j]))

def check_make(path_check):
    """
    Convenience routine to avoid that annoying 'can't make directory; already
    present!' error.
    """
    if not path.exists(path_check):
        makedirs(path_check)

def wizard(config):
    """Run the wizard code, given a configuration dictionary

    Parameters
    ----------
    config : the dictionary of config files

    """
    from time import time
    if config['verbose']:
        time0 = time()
    else:
        time0 = 0

    for entry_index, entry in enumerate(config['run']):
        if entry == 'make_directories':
            if time0:
                print('make directories', time() - time0)
            make_directories(config)
        elif entry == 'dataset':
            if time0:
                print('dataset', time() - time0)
                if config['verbose'] > 1:
                    time0i = time()
                else:
                    time0i = 0
            reference_catalogs, unknown_catalogs = dataset(time0=time0i, **config['dataset'])
        elif entry == 'paircounts':
            """
            We need from datasets:
            reference_catalogs [output form dataset function]
            unknown_catalogs [output from dataset function]
            reference_bins [output from config]
            unknown_bins [output from config]
            """
            if time0:
                print('paircounts', time() - time0)
                if config['verbose'] > 1:
                    time0i = time()
                else:
                    time0i = 0
            pairs = paircounts(reference_catalogs, config['dataset']['reference_bins'],
                               unknown_catalogs, config['dataset']['unknown_bins'],
                               config['dataset']['healpix_nside'], time0=time0i, **config['paircounts'])
        elif entry == 'combine':
            """
            pairs and healpix_nside come from previous run of paircounts
            """
            if time0:
                print('combine', time() - time0)
                if config['verbose'] > 1:
                    time0i = time()
                else:
                    time0i = 0
            if 'paircounts' not in config['run']:
                # need to come up with pairs as a default
                pairs = {'DuDr': None, 'RuRr': None, 'DuRr': None, 'RuDr': None}
            combined_pairs = combine(reference_catalogs, config['dataset']['reference_bins'], unknown_catalogs, config['dataset']['unknown_bins'], pairs, config['dataset']['healpix_nside'], time0=time0i, **config['combine'])
        elif entry == 'dndz':
            if time0:
                print('dndz', time() - time0)
                if config['verbose'] > 1:
                    time0i = time()
                else:
                    time0i = 0
            if 'combine' not in config['run']:
                # put in false combined_pairs since it probably doesn't matter
                combined_pairs = None
            min_sep = config['paircounts']['min_sep']
            max_sep = config['paircounts']['max_sep']
            nbins = config['paircounts']['nbins']
            redshift_bins = np.arange(*config['dataset']['reference_bins'][config['dndz']['redshift_column']][1])
            redshifts = np.append(-10, np.append(0.5 * (redshift_bins[1:] + redshift_bins[:-1]), 10))
            wz, cov, cov_full, phi, stats, tags = dndz(combined_pairs, redshifts, redshift_bins, min_sep=min_sep, max_sep=max_sep, nbins=nbins, time0=time0i, **config['dndz'])
        elif entry == 'plot':
            # plot the results
            if time0:
                print('plot', time() - time0)
                if config['verbose'] > 1:
                    time0i = time()
                else:
                    time0i = 0
            z_min = config['dndz']['z_min']
            z_max = config['dndz']['z_max']

            # redshift_column = config['dndz']['redshift_column']
            plot(phi, cov_full, stats, tags, unknown_catalogs[0], redshift_bins,
                 z_min=z_min, z_max=z_max, #redshift_column=redshift_column,
                 time0=time0i,
                 **config['plot'])

        elif entry == 'compare':
            # plot the results
            if time0:
                print('compare', time() - time0)
                if config['verbose'] > 1:
                    time0i = time()
                else:
                    time0i = 0
            # make sure the redshift bins are the right scales
            redshift_bins_conds = (redshift_bins >= z_min) & (redshift_bins < z_max)
            # make sure the length for redshift_bins makes sense
            assert len(redshift_bins[redshift_bins_conds]) == len(wz[0]) + 1,'redshift bins are length {0} but wz + 1 is length {1}'.format(len(redshift_bins[redshift_bins_conds]), len(wz[0]) + 1)
            # fit to another catalog
            wz_corrected, cov_corrected, pz_corrected, redshiftbias = \
                compare(wz=wz, cov=cov,
                        unknown_bins=config['dataset']['unknown_bins'],
                        unknown_catalog=unknown_catalogs[0],
                        redshift_bins=redshift_bins[redshift_bins_conds],
                        time0=time0i,
                        **config['compare'])
            # correct unknown_catalogs[0]
            keys = unknown_catalogs[0].groupby(tags).groups.keys()
            cat = unknown_catalogs[0]
            for key_i, key in enumerate(keys):
                conds = (cat[tags] == key).all(axis=1)
                cat.loc[conds,config['compare']['redshift_catalog_column'] + '_CORRECTED'] = cat.loc[conds,config['compare']['redshift_catalog_column']] - redshiftbias.median_sample()[2 * key_i]
            unknown_catalogs[0] = cat
            # correct the phi object as well
            phi['phi'] = np.hstack(wz_corrected)
            if redshiftbias.cov_mode == 'full':
                phi['phi_err'] = np.sqrt(np.diag(cov_corrected))
            elif redshiftbias.cov_mode == 'tomo':
                phi['phi_err'] = np.sqrt(np.hstack([np.diag(cov_corrected_i) for cov_corrected_i in cov_corrected]))
            elif redshiftbias.cov_mode == 'diagonal':
                phi['phi_err'] = np.sqrt(np.hstack(cov_corrected))

            # if we want to do plots again after correction, change plot param
            if 'plot' in config['run'][entry_index:]:
                if 'plot_path' in config['plot']:
                    config['plot']['plot_path'] = config['compare']['compare_path']
                    check_make(path.dirname(config['plot']['plot_path']))
                    # also update the columns so that we can plot the new
                    # corrected PZ
                    config['plot']['redshift_catalog_columns'].append(config['compare']['redshift_catalog_column'] + '_CORRECTED')

            # fit to another catalog

            # repeat plotting
            pass
        elif entry == 'compare_noz_fom':
                if time0:
                    print('compare_noz_fom', time() - time0)
                    if config['verbose'] > 1:
                            time0i = time()
                    else:
                            time0i = 0
                z_min = config['dndz']['z_min']
                z_max = config['dndz']['z_max']
                path_rec=config['dndz']['dndz_path']
                biaised_nz=compare_noz_fom(phi,tags,unknown_catalogs[0], redshift_bins,
                z_min=z_min, z_max=z_max, path_rec=path_rec,#redshift_column=redshift_column,
                time0=time0,
                **config['compare_noz_fom'])
        elif entry == 'FoM':
                if time0:
                    print('compare_noz_fom', time() - time0)
                    if config['verbose'] > 1:
                            time0i = time()
                    else:
                            time0i = 0
                cosmosissource = 'source my-source'
                Nz_path=config['compare_noz_fom']['compare_noz_fom_path']
                fom=FoM(Nz_path=Nz_path,time0=time0,**config['FoM'])
        elif entry == 'plot_FoM':
                if time0:
                    print('compare_noz_fom', time() - time0)
                    if config['verbose'] > 1:
                            time0i = time()
                    else:
                            time0i = 0
                fom_path=config['FoM']['FoM_path']
                comp=config['compare_noz_fom']['redshift_catalog_columns']
                bins=config['dataset']['unknown_bins']
                plot_fom=plot_FoM(fom_path=fom_path,comp=comp,bins=bins,time0=time0,**config['plot_FoM'])

    if time0:
        print('done', time() - time0)


import keras.backend as K

def mitos_fscore(y_true, y_pred):
    """
    Calculate the f1 score based on the class 0 (mitosis). It uses the
    functions for tensors provided by Tensorflow/Theano. And can be used as
    metric for keras model
    :param y_true: True labels. Theano/TensorFlow tensor.
    :param y_pred: False labels. Theano/TensorFlow tensor.
    :return: f1 score as a tensor
    """
    # true_output = K.argmax(y_true)
    # pred_output = K.argmax(y_pred)
    true_output = y_true
    pred_output = K.round(y_pred)
    index_true_class = K.equal(true_output, 0)
    floatx = K.floatx()
    if K.backend() == 'theano':
        # compares index_true_class == false
        inverted_true_class = K.equal(index_true_class, 0)
        index_pred_class = K.equal(pred_output, 0)
    else:
        inverted_true_class =  K.equal(index_true_class, False)
        inverted_true_class = K.cast(inverted_true_class, floatx)
        index_pred_class = K.equal(pred_output, False)
        index_pred_class = K.cast(index_pred_class, floatx)
        index_true_class = K.cast(index_true_class, floatx)

    false_positive = K.sum(inverted_true_class * index_pred_class)
    true_positives = K.sum(K.cast(K.equal(true_output, pred_output), floatx)* index_true_class)
    false_negatives = K.sum(index_true_class) - true_positives

    precision = true_positives / (true_positives + false_positive + K.epsilon())
    recall = true_positives / (true_positives + false_negatives+K.epsilon())
    return 2 * (precision * recall)/(precision + recall+K.epsilon())
    #print(K.eval(fscore))

def fscore(y_true, y_pred, not_extracted = 0):
    from sklearn.metrics import confusion_matrix

    epsilon = 1e-07
    conf_mat = confusion_matrix(y_true=y_true, y_pred=y_pred)
    true_positives = conf_mat[0,0]
    false_negatives = conf_mat[0,1] + not_extracted
    false_positives = conf_mat[1,0]

    precision = true_positives / (true_positives + false_positives + epsilon)
    recall = true_positives / (true_positives + false_negatives + epsilon)
    return 2 * (precision * recall)/(precision + recall + epsilon)

def print_conf_matrix(y_true, y_pred, not_extracted=0):
    from sklearn.metrics import confusion_matrix

    conf_mat = confusion_matrix(y_true=y_true, y_pred=y_pred)
    true_positives = conf_mat[0, 0]
    false_negatives = conf_mat[0, 1] + not_extracted
    false_positives = conf_mat[1, 0]
    true_negatives = conf_mat[1,1]

    print('\t\tMitosis\tNo-mitosis')
    print('Mitosis\t\t{}\t{}'.format(true_positives, false_negatives))
    print('No-mitosis\t{}\t{}'.format(false_positives, true_negatives))

if __name__ == '__main__':
    m1 = [[1,0],[1,0],[0,1], [1,0], [0,1], [0,1], [1,0],[0,1],[1,0],[1,0],[1,0]]
    m2 = [[1,0], [1,0], [1,0], [0,1], [0,1], [0,1], [1,0], [1,0],[1,0],[0 ,1],[0,1]]

    import numpy as np
    pred = K.variable(m1)
    true = K.variable(m2)
    res = K.eval(mitos_fscore(true, pred))
    print(np.argmax(m2, 1))
    print(res)
    i=0


import cv2
import numpy as np

def Add_gaussian_Noise(image,mean,sigma):
    noiseArr = image.copy()
    noiseArr = np.random.normal(mean,sigma,image.shape)
    np.add(image,noiseArr,image,casting="unsafe")
    return;

def Add_salt_pepper_Noise(image,pa,pb):
    row,col,ch=image.shape
    blackspot=row*col*pa
    whitespot=row*col*pb
    for i in range(int(blackspot)):
        image[int(np.random.uniform(0,row))][int(np.random.uniform(0,col))]=0
    for i in range(int(whitespot)):
        image[int(np.random.uniform(0,row))][int(np.random.uniform(0,col))]=255

def main():

    image=cv2.imread('OpenCV_homework/Test_images/baboon.jpg')
    cv2.namedWindow('Original image')
    cv2.imshow('Original',image)

#gaussian_Noise
    noise_img=image.copy()
    mean=0
    sigma=50
    Add_gaussian_Noise(noise_img,mean,sigma)
    cv2.imshow('Gaussian Noise',noise_img)

    noise_dst=noise_img.copy()
    cv2.blur(noise_dst,(3,3))
    cv2.imshow('Box filter',noise_dst)

    noise_dst1=noise_img.copy()
    cv2.GaussianBlur(noise_dst1,(3,3),1.5)
    cv2.imshow('GaussianBlur filter',noise_dst1)

    noise_dst2=noise_img.copy()
    cv2.medianBlur(noise_dst2,3)
    cv2.imshow('Median filter',noise_dst2)

#salt_pepper_Noise
    noise_img2=image.copy()
    pa=0.01
    pb=0.01
    Add_salt_pepper_Noise(noise_img2,pa,pb)
    cv2.imshow("Salt and Peper Noise", noise_img2)

    noise_dst3=noise_img2.copy()
    cv2.blur(noise_dst3,(3,3))
    cv2.imshow('Box filter2',noise_dst3)

    noise_dst4=noise_img2.copy()
    cv2.GaussianBlur(noise_dst4,(3,3),1.5)
    cv2.imshow('GaussianBlur filter2',noise_dst4)

    noise_dst5=noise_img2.copy()
    cv2.medianBlur(noise_dst5,3)
    cv2.imshow('Medianfilter2',noise_dst5)

    print('MEAN:',mean)
    print('SIGMA:',sigma)
    print('Pa:',pa)
    print('Pb:',pb)
    print('Best Kernel Size: 3x3')

    cv2.waitKey(0)

if __name__ == "__main__":
    main()


import logging
import os

LOG_FORMAT = '%(asctime)s %(message)s'
DEFAULT_LEVEL = logging.DEBUG

class LogWrapper():
    def __init__(self, name, mode="w"):
        self.create_directory()
        self.filename = './logs/' + name + '.log'
        self.logger = logging.getLogger(name)
        self.logger.setLevel(DEFAULT_LEVEL)

        file_handler = logging.FileHandler(self.filename, mode=mode)
        formatter = logging.Formatter(LOG_FORMAT, datefmt='%Y-%m-%d %H:%M:%S')
        file_handler.setFormatter(formatter)
        self.logger.addHandler(file_handler)
        self.logger.info("LogWapper init() " + self.filename)

    def create_directory(self):
        path = './logs'
        if not os.path.exists(path):
            os.makedirs(path)


from __future__ import print_function, division
import os
import time
import math
import tensorflow as tf
import numpy as np
import glob
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from scipy import misc
import utils
import LossLib

plt.rcParams['figure.figsize'] = (10.0, 8.0)  # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'

batch_size = 100
img_h = 218
img_w = 178
img_c = 3
w_to_h = 1.0 * img_w / img_h
x_dim = 116412  # 218, 178, 3 dimension of each image
noise_dim = 64

img_dir = "./data/img_align_celeba/"
out_dir = "out"
prefix = "deconv-g-conv-d-vanilla-celeba"
save_dir = "save"
save_dir_prefix = save_dir + "/" + prefix
logs_path = "logs/" + prefix

def load_images(img_dir):
    img_paths = []
    for img in os.listdir(img_dir):
        img_paths.append(os.path.join(img_dir, img))
    total = len(img_paths)
    i = 0
    while (True):
        if i + batch_size >= total:
            i = 0
            continue
        images = []
        for j in range(batch_size):
            images.append(misc.imread(img_paths[i + j]))
        images = np.reshape(np.asarray(images), [-1, x_dim])
        images = utils.preprocess_img_rgb(images)
        assert not np.any(np.isnan(images)), "Images should not contain nan's"
        yield(images)
        i = (i + batch_size) % total

def mkdir_p(dir):
    if not os.path.exists(dir):
        os.makedirs(dir)

def save_images(dir, images, it):
    fig = plt.figure(figsize=(10 * w_to_h, 10))
    gs = gridspec.GridSpec(10, 10)
    gs.update(left=0, right=1, top=1, bottom=0, wspace=0, hspace=0)

    for i, img in enumerate(images):
        ax = plt.subplot(gs[i])
        plt.axis('off')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        # ax.set_aspect('equal')
        img = utils.deprocess_img(img)
        plt.imshow(img.reshape([img_h, img_w, img_c]))

    imgpath = dir + "/" + str(it).zfill(10) + ".jpg"
    print("Saving img " + imgpath)
    fig.savefig(imgpath)
    plt.close(fig)

def get_session():
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    session = tf.Session(config=config)
    return session

def leaky_relu(x, alpha=0.01):
    return tf.maximum(alpha * x, x)

def sample_z(m, n):
    return np.random.uniform(-1., 1., size=[m, n])

def discriminator(x):
    with tf.variable_scope("discriminator"):
        # Cluster 1
        fc0 = tf.layers.dense(inputs=x, units=16 * 16, activation=leaky_relu)
        rs0 = tf.reshape(fc0, [-1, 16, 16, 1])
        c1 = tf.layers.conv2d(inputs=rs0, filters=16, kernel_size=5, strides=1, padding='same', activation=leaky_relu)
        c2 = tf.layers.conv2d(inputs=c1, filters=16, kernel_size=5, strides=1, padding='same', activation=leaky_relu)
        rs3 = tf.reshape(c2, [-1, 16 * 16 * 16])
        # Cluster 2
        fc4 = tf.layers.dense(inputs=rs3, units=16 * 16, activation=leaky_relu)
        rs4 = tf.reshape(fc4, [-1, 16, 16, 1])
        c5 = tf.layers.conv2d(inputs=rs4, filters=16, kernel_size=5, strides=1, padding='same', activation=leaky_relu)
        c6 = tf.layers.conv2d(inputs=c5, filters=16, kernel_size=5, strides=1, padding='same', activation=leaky_relu)
        rs7 = tf.reshape(c6, [-1, 16 * 16 * 16])
        # Tail cluster 3
        fc7 = tf.layers.dense(inputs=rs7, units=16 * 16, activation=leaky_relu)
        logits = tf.layers.dense(inputs=fc7, units=1)
        return logits

def generator(z, keep_prob):
    with tf.variable_scope("generator"):
        fc0 = tf.layers.dense(inputs=z, units=1024, activation=leaky_relu)
        bn0 = tf.layers.batch_normalization(fc0, axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, training=True)

        in_channels_1 = 128
        fc1 = tf.layers.dense(inputs=bn0, units=7 * 7 * in_channels_1, activation=leaky_relu)
        bn1 = tf.layers.batch_normalization(fc1, axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, training=True)
        rs1 = tf.reshape(bn1, [-1, 7, 7, in_channels_1])

        in_channels_2 = 64
        ct2_filter = tf.get_variable("ct2_filter", shape=(4, 4, in_channels_2, in_channels_1), dtype=tf.float32, initializer=tf.truncated_normal_initializer(), trainable=True)
        ct2 = tf.nn.relu(tf.nn.conv2d_transpose(rs1, filter=ct2_filter, output_shape=(batch_size, 14, 14, in_channels_2), strides=(1, 2, 2, 1), padding='SAME', data_format='NHWC'))
        bn2 = tf.layers.batch_normalization(ct2, axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, training=True)

        ct3_filter = tf.get_variable("ct3_filter", shape=(4, 4, 1, in_channels_2), dtype=tf.float32, initializer=tf.truncated_normal_initializer(), trainable=True)
        ct3 = tf.nn.relu(tf.nn.conv2d_transpose(bn2, filter=ct3_filter, output_shape=(batch_size, 28, 28, 1), strides=(1, 2, 2, 1), padding='SAME', data_format='NHWC'))

        rs4 = tf.reshape(ct3, [-1, 28 * 28])
        img = tf.layers.dense(inputs=rs4, units=x_dim, activation=tf.tanh)
        return img

def log(x):
    return tf.log(x + 1e-8)

tf.reset_default_graph()

with tf.name_scope('input'):
    x = tf.placeholder(tf.float32, shape=[None, x_dim], name="x-input")
    z = tf.placeholder(tf.float32, shape=[None, noise_dim], name="z-input")
    keep_prob = tf.placeholder(tf.float32, name='keep_prob')

with tf.variable_scope("") as scope:
    G_sample = generator(z, keep_prob)
    D_real = discriminator(x)
    scope.reuse_variables()
    D_fake = discriminator(G_sample)

D_loss, G_loss = LossLib.VanillaGANLoss(D_real, D_fake)

dlr, glr, beta1 = 1e-3, 1e-3, 0.5
D_solver = tf.train.AdamOptimizer(learning_rate=dlr, beta1=beta1)
G_solver = tf.train.AdamOptimizer(learning_rate=glr, beta1=beta1)
D_extra_step = tf.get_collection(tf.GraphKeys.UPDATE_OPS, 'discriminator')
G_extra_step = tf.get_collection(tf.GraphKeys.UPDATE_OPS, 'generator')
D_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'discriminator')
G_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'generator')
with tf.control_dependencies(D_extra_step):
    D_train_step = D_solver.minimize(D_loss, var_list=D_vars)
with tf.control_dependencies(G_extra_step):
    G_train_step = G_solver.minimize(G_loss, var_list=G_vars)

mkdir_p(out_dir)
mkdir_p(save_dir)

tf.summary.scalar("D_loss", D_loss)
tf.summary.scalar("G_loss", G_loss)
summary_op = tf.summary.merge_all()
writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())

def train(sess, G_train_step, G_loss, D_train_step, D_loss, D_extra_step, G_extra_step, save_img_every=250, print_every=50, max_iter=1000000):
    Saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=1)
    if glob.glob(save_dir + "/*"):
        Saver.restore(sess, tf.train.latest_checkpoint(save_dir))

    batches = load_images(img_dir)
    t = time.time()
    for it in range(max_iter):
        xmb = next(batches)
        z_noise = sample_z(batch_size, noise_dim)

        if it % save_img_every == 0:
            samples = sess.run(G_sample, feed_dict={x: xmb, z: z_noise, keep_prob: 1.0})
            save_images(out_dir, samples[:100], it)

        _, D_loss_curr, summary, _ = sess.run([D_train_step, D_loss, summary_op, D_extra_step], feed_dict={x: xmb, z: z_noise, keep_prob: 0.3})
        _, G_loss_curr, _ = sess.run([G_train_step, G_loss, G_extra_step], feed_dict={x: xmb, z: z_noise, keep_prob: 0.3})

        if math.isnan(D_loss_curr) or math.isnan(G_loss_curr):
            print("D or G loss is nan", D_loss_curr, G_loss_curr)
            exit()

        if it % print_every == 0:
            print('Iter: {}, D: {:.4}, G: {:.4}, Elapsed: {:.4}'.format(it, D_loss_curr, G_loss_curr, time.time() - t))
            t = time.time()

        if it % 10 == 0:
            writer.add_summary(summary, global_step=it)
        if it % 100 == 0:
            Saver.save(sess, save_dir_prefix, global_step=it)

with get_session() as sess:
    sess.run(tf.global_variables_initializer())
    train(sess, G_train_step, G_loss, D_train_step, D_loss, D_extra_step, G_extra_step, save_img_every=25, print_every=1, max_iter=1000000)


#!/usr/bin/env python

import urllib2, json

class Travis:
    def __init__(self, endpoint, token):
        self.endpoint = endpoint
        self.token = token

    def _api_call(self, resource, data=None):
        url = self.endpoint + resource
        if data:
            data = json.dumps(data)

        req = urllib2.Request(url, data)
        req.add_header('Authorization', 'token ' + self.token)

        if data:
            req.add_header('Content-Type', 'application/json; charset=UTF-8')

        p = urllib2.urlopen(req)
        return json.loads(p.read())

    def get_builds(self, repository, branch = None):
        builds = self._api_call('/repos/{}/builds'.format(repository))
        if branch:
            return filter(lambda build : build['branch'] == branch, builds)
        else:
            return builds

    def build(self, build_id):
        return self._api_call('requests', { 'build_id': build_id })['flash'][0]


import mmap

from .dictionaryheader import DictionaryHeader
from .dictionaryversion import is_dictionary
from .doublearraytrie import DoubleArrayTrie

class BinaryDictionary(object):

    def __init__(self, bytes_, header, trie):
        self._bytes = bytes_
        self._header = header
        self._trie = trie

    @staticmethod
    def _read_dictionary(filename, access=mmap.ACCESS_READ):
        with open(filename, 'rb') as system_dic:
            bytes_ = mmap.mmap(system_dic.fileno(), 0, access=access)
        offset = 0

        header = DictionaryHeader.from_bytes(bytes_, offset)
        offset += header.storage_size()

        if not is_dictionary(header.version):
            raise Exception('invalid dictionary version')

        trie = DoubleArrayTrie(bytes_, offset)
        offset += trie.storage_size()

        return bytes_, header, trie

    @classmethod
    def from_system_dictionary(cls, filename):
        args = cls._read_dictionary(filename)
        version = args[2].version
        if not is_dictionary(version):
            raise IOError('invalid system dictionary')
        return cls(*args)

    def close(self):
        del self._trie
        self._bytes.close()

    @property
    def header(self) -> DictionaryHeader:
        """

        :return:
        :rtype: DictionaryHeader
        """
        return self._header


#!/usr/bin/env python3

#https://codeforces.com/problemset/problem/180/F
#1e5所以不能一个个比, 否则(N**2)的复杂度
#复杂度的例子!

def f(l1,l2):
    n  = len(l1)
    t1 = [(l1[i],i) for i in range(n)]
    t2 = [(l2[i],i) for i in range(n)]
    t1.sort(key=lambda t:t[0])
    t2.sort(key=lambda t:t[0])
    p  = [0]*n
    for i in range(n):
        p[t2[i][1]]=t1[i][1]+1
    return p

q  = int(input())
l1 = list(map(int,input().split())) #1e5
l2 = list(map(int,input().split())) #1e5
print(*f(l1,l2))


# 源自：https://blog.csdn.net/hitzijiyingcai/article/details/81152174
词语 = '用中文编程'
所有行 = []
for 行数 in range(15, -15, -1):
    一行 = ""
    for 列数 in range(-15, 15):
        列 = 列数 * 0.12
        行 = 行数 * 0.12
        字 = ""
        if (列 ** 2 + 行 ** 2 - 1) ** 3 <= 列 ** 2 * 行 ** 3:
            字 = 词语[(列数 - 行数) % len(词语)]
        else:
            字 = '一'
        一行 += 字
    所有行.append(一行)
print('\n'.join(所有行))


# does not support long int

def isnumeric (x):
    return isinstance(x, int) or isinstance(x, float)

def overlap(x1=0, x2=0, x3=0, x4=0):
    if isnumeric(x1) and isnumeric(x2) and isnumeric(x3) and isnumeric(x4):
        x1 = abs(x1)
        x2 = abs(x2)
        x3 = abs(x3)
        x4 = abs(x4)

        if x1 > x3:
            p1 = x1 <= x4
        else:
            p1 = x3 <= x2
        if x2 > x4:
            p2 = x4 >= x1
        else:
            p2 = x2 >= x3
    else:
        raise Exception("coordinates format error")

    return p1 or p2

# quick demo of functionality:

print(overlap(-0.1, -4.2, -4.1, -5))
print(overlap(0.1, 4.2, 4.1, 5))
print(overlap(0, 0, 0, 0))
print(overlap())                # default
print(overlap(0, 3, 4, 5))
print(overlap(-0, -3, -4, -5))
# print(overlap("gfl", -3, -4, -5))


from django.forms import ModelForm, Textarea, TextInput, DateInput, TimeInput
from django.utils.safestring import mark_safe

from dal import autocomplete

from .models import CourseProject

class CourseProjectForm(ModelForm):
    class Meta:
        model = CourseProject
        fields = [
            'title', 'description', 'student', 'supervisor',
            'mark', 'text', 'slides', 'materials', 'course_semester'
        ]
        widgets = {
            'title': TextInput(attrs={'size': '100'}),
            'description': Textarea(attrs={'data-uk-htmleditor': '{markdown:true}'}),
            'student': autocomplete.ModelSelect2(url='students:autocomplete'),
        }


#!/usr/bin/python3

from classes import *
import os

#================= 1 : Clause unitaire ====================

def unitaire(clause, val):
    """
    Teste si une clause est unitaire (si tous ses littéraux sauf 1 sont à faux)
    Renvoie le littéral imposé si oui, 0 si non
    """
    resu=0
    unitaire=1
    ind=-1
    for i in range(len(clause)):
        x=clause[i]
        if val[abs(x)]*x>=0 and resu==0:
            # Premier pas faux rencontré
            resu=x
            ind=i
        elif val[abs(x)]*x>=0:
            # Rencontre d'un autre pas faux
            unitaire = 0
    return unitaire*resu, ind

#================= 2 : Polarité unique ====================

def cherche_polarite_unique(clauses, v):
    """
    Cherche une variable apparaissant toujours avec la même polarité dans les
    clauses courantes.
    Renvoie la liste des littéraux correspondants (éventuellement vide).
    """
    tab=[0]*(v+1)
    for cond in clauses:
        if cond.active:
            for x in cond:
                if tab[abs(x)]==0:
                    tab[abs(x)]=x
                elif tab[abs(x)]!=x:
                    tab[abs(x)]=v+1
    l=[]
    for x in range(1,v+1):
        if tab[x]!=v+1:
            l.append(tab[x] if tab[x]!=0 else x)
    return l

#print(cherche_polarite_unique([[1,2,3],[-1,2,4],[2,-4],[-3]],5))

#================= 3 : Pré-traitement =====================

def pre_trait(cond,v):
    """
    Effectue le pré-traitement :
        - Si la clause est vide, renvoie 0 (=> blocage)
        - Si elle est tautologique, renvoie 1 (=> clause inutile)
        - Si elle n'est rien de tout ça, renvoie 2 (=> rien à faire)
    """
    l=len(cond)
    if l==0:
        return 0
    else:
        tab=[0]*(v+1)
        for i in cond:
            if tab[abs(i)]==0:
                tab[abs(i)]=i
            elif tab[abs(i)]!=i:
                return 1
        return 2

#================= 4 : Apprentissage de clauses ===========

def cl_uip(gr):
    """Calcule un ensemble UIP pour le graphe donné en entrée."""
    violets = {"conflit"}
    # Futurs sommets violets (on retire "conflit" à la fin)
    jaunes = {x for x in gr.peres("conflit") if gr[x][2] != None}
    # Contient tous les pères des sommets violet : au moment où il n'y en a plus
    # qu'un, c'est nécessairement le jaune qu'on cherche
    while len(jaunes) != 1:
        for u in jaunes:
            # On cherche le prochain sommet dont tous les fils ont été vus
            b = True
            for v in gr.fils(u):
                if v not in violets:
                    b = False
                    break
            if b:
                break
        violets.add(u)
        jaunes.remove(u)
        for v in gr.peres(u):
            if gr[v][2] != None:
                jaunes.add(v)
    violets.remove("conflit")
    return(jaunes.pop(), violets)

def cl_graphe(clauses, cl, pile, niv):
    """Génère le graphe lié à un conflit."""
    bleus = {ac.num : ac for ac in pile if ac.niv == niv
                                        and ac.acte != 'validation'}
    queue = []
    # Construction récursive du graphe
    g = Graphe()
    g.aj_s("conflit".format(cl), "crimson")
    for l in clauses[cl]:
        queue.append((-l, "conflit"))
    while queue != []:
        u, v = queue[0]
        g.aj_s(u)
        g.aj_a(u, v)
        if abs(u) in bleus:
            g.colorie(u, 'dodgerblue3')
            for uu in bleus[abs(u)].peres:
                if (uu, u) not in queue:
                    queue.append((uu, u))
                    #print(u, uu)
        queue = queue[1:]
    # Recherche de l'ensemble UIP
    jaune, violets = cl_uip(g)
    g.colorie(jaune, 'gold')
    for v in violets:
        g.colorie(v, 'mediumpurple1')

    return(g)

def cl_apprend(graphe, pile, niv):
    """Donne la clause à apprendre et le niveau auquel revenir."""
    cl_app = Clause()
    for v in graphe:
        if graphe[v][2] in ["mediumpurple1", "crimson"]:
            for u in graphe.peres(v):
                if graphe[u][2] != "mediumpurple1":
                    cl_app.append(-u)
    niveaux = {abs(u) : [] for u in cl_app if graphe[-u][2] == None}
    for ac in pile:
        if ac.num in cl_app and graphe[-ac.num][2] == None and ac.niv != niv:
            niveaux[ac.num].append(ac.niv)
        if -ac.num in cl_app and graphe[ac.num][2] == None and ac.niv != niv:
            niveaux[ac.num].append(ac.niv)
    try:
        niv_app = max([max(niveaux[u]) for u in niveaux])
    except ValueError:
        niv_app = 0
    return(cl_app, niv_app)

def cl_interactive_mode(clauses, cl, pile, niv):
    """Mode interactif pour l'apprentissage de clauses."""
    for i in range(3):
        r = input("Conflit dans la clause {0} ! Regardez le graphe (g), "
                  "continuez (c) ou tuez le mode interactif (t).".format(cl))
        gr = cl_graphe(clauses, cl, pile, niv)
        app, niv_app = cl_apprend(gr, pile, niv)
        if r == 'g':
            gr.export(".rendu_graphe", True)
            print("Le graphe a été généré. La clause à apprendre est :")
            print(app)
            print(niv_app)
            input("Faites Entrée pour poursuivre.")
            os.system("rm --preserve-root ./{0}*".format(".rendu_graphe"))
            return(True, app, niv_app)
        elif r == 'c':
            return(True, app, niv_app)
        elif r == 't':
            return(False, app, niv_app)
    raise ValueError("Trois commandes erronnées successives.")

def cl_mute_mode(clauses, cl, pile, niv):
    """Mode silencieux pour l'apprentissage de clauses."""
    gr = cl_graphe(clauses, cl, pile, niv)
    app, niv_app = cl_apprend(gr, pile, niv)
    return(app, niv_app)


import logging
import os
import shutil
from functools import lru_cache

import regex

from doc_curation.md.file import MdFile
from doc_curation.md.library import metadata_helper
from indic_transliteration import sanscript

def defolderify_single_md_dirs(dir_path, dry_run=False):
  while dir_path.endswith("/"):
    dir_path = dir_path[:-1]
  contents = os.listdir(dir_path)
  for possible_subdir in contents:
    subpath = os.path.join(dir_path, possible_subdir)
    if os.path.isdir(subpath):
      defolderify_single_md_dirs(dir_path=subpath, dry_run=dry_run)
  if len(contents) != 1:
    return
  if not contents[0] == "_index.md":
    return
  index_file_path = os.path.join(dir_path, contents[0])
  dest_file_path = dir_path + ".md"
  logging.info("Moving %s to %s", str(index_file_path), dest_file_path)
  if not dry_run:
    os.rename(src=index_file_path, dst=dest_file_path)
    md_file = MdFile(file_path=dest_file_path)
    (metadata, _) = md_file.read()
    title = metadata["title"]
    if title.startswith("+"):
      title = title[1:]
    md_file.set_title(title=title, dry_run=False)
    os.rmdir(dir_path)

def shift_contents(dir_path, substitute_content_offset, start_index=None, end_index=None, index_position=0, replacer=lambda md_file, content: md_file.replace_content_metadata(new_content=content, dry_run=False)):
  files = [os.path.join(dir_path, x) for x in os.listdir(dir_path) if x != "_index.md" and x.endswith(".md")]
  files.sort()
  index_to_content_original = {}
  index_to_md_file = get_index_to_md(dir_path=dir_path, index_position=index_position)

  for index, md_file in index_to_md_file.items():
    (_, content) = md_file.read()
    index_to_content_original[index] = content

  min_index = min(index_to_content_original.keys())
  max_index = max(index_to_content_original.keys())
  if start_index is None:
    start_index = min_index
  if end_index is None:
    end_index = max_index
  for index, content in index_to_content_original.items():
    if start_index <= index and end_index >= index:
      offset_index = index + substitute_content_offset
      logging.info("Shifting %d to %d", offset_index, index)
      if offset_index in index_to_content_original.keys():
        content = index_to_content_original[offset_index]
        md_file = index_to_md_file[index]
        replacer(md_file, content)

def shift_indices(dir_path, new_index_offset, start_index=1, end_index=9999, index_position=0, dry_run=False):
  files = [os.path.join(dir_path, x) for x in os.listdir(dir_path) if x != "_index.md" and x.endswith(".md")]
  files.sort()
  index_to_md_file = get_index_to_md(dir_path=dir_path, index_position=index_position)
  for index, md_file in index_to_md_file.items():
    if start_index <= index and end_index >= index:
      new_index = index + new_index_offset
      name_parts = os.path.basename(md_file.file_path).split("_")
      index_pattern = "%%0%dd" % len(name_parts[index_position])
      name_parts[index_position] = index_pattern % new_index
      new_file_path = os.path.join(os.path.dirname(md_file.file_path), "_".join(name_parts))
      logging.info("Shifting %d to %d, %s to %s", index, new_index, md_file.file_path, new_file_path)
      if not dry_run:
        shutil.move(md_file.file_path, new_file_path)

def remove_file_by_index(dir_path, indices, index_position=0):
  index_to_md_file = get_index_to_md(dir_path=dir_path, index_position=index_position)
  for index, md_file in index_to_md_file.items():
    if index in indices:
      os.remove(md_file.file_path)

@lru_cache(maxsize=2)
def get_sub_path_to_reference_map(ref_dir, sub_path_id_maker=None):
  ref_md_files = get_md_files_from_path(dir_path=ref_dir)
  sub_path_to_reference = {}
  if sub_path_id_maker is None:
    sub_path_id_maker = lambda x: get_sub_path_id(sub_path=str(x).replace(ref_dir, ""))
  for md_file in ref_md_files:
    sub_path_id = sub_path_id_maker(md_file.file_path)
    if sub_path_id is not None:
      sub_path_to_reference[sub_path_id] = md_file
  return sub_path_to_reference

def get_sub_path_id(sub_path, basename_id_pattern=r"(.+?)(?=[_\.]|$)"):
  id_parts = []
  for name in sub_path.split("/"):
    if name == "":
      continue
    elif name == "_index.md":
      # id_parts.append(name)
      pass
    else:
      base_id_match = regex.search(basename_id_pattern, name)
      if base_id_match is None:
        logging.fatal("No match in %s", sub_path)
      id_parts.append(base_id_match.group(1))
  return "/".join(id_parts)

def get_index_to_md(dir_path, index_position=0):
  files = [os.path.join(dir_path, x) for x in os.listdir(dir_path) if x != "_index.md" and x.endswith(".md")]
  files.sort()
  index_to_md_file = {}
  for index, file_path in enumerate(files):
    base_name = os.path.basename(file_path)
    try:
      index = int(base_name.replace(".md", "").split("_")[index_position])
    except ValueError:
      logging.warning(f"Skipping file with irregular index- {file_path}")
    md_file = MdFile(file_path=file_path)
    index_to_md_file[index] = md_file
  return index_to_md_file

def get_md_files_from_path(dir_path, file_pattern="**/*.md", file_name_filter=lambda x: True):
  from pathlib import Path
  # logging.debug(list(Path(dir_path).glob(file_pattern)))
  md_file_paths = sorted(filter(file_name_filter, Path(dir_path).glob(file_pattern)))
  md_file_paths = [f for f in md_file_paths if os.path.isfile(f)]
  return [MdFile(path) for path in md_file_paths]

def migrate(files, location_computer, dry_run=False):
  """Migrate a bunch of files to a new location (dynamically computed by location_computer function.)"""
  logging.info("Processing %d files", len(files))
  for f in files:
    new_path = location_computer(str(f))
    if new_path is not None:
      logging.info("Moving %s to %s", str(f), new_path)
      if not dry_run:
        os.makedirs(os.path.dirname(new_path), exist_ok=True)
        os.rename(src=f, dst=new_path)

def fix_index_files(dir_path, frontmatter_type=MdFile.TOML, transliteration_target=sanscript.DEVANAGARI, overwrite=False, dry_run=False):
  logging.info("Fixing index files")
  # Get all non hidden directories.
  dirs = [x[0] for x in os.walk(dir_path) if "/." not in x[0]]
  # set([os.path.dirname(path) for path in Path(dir_path).glob("**/")])
  for dir in dirs:
    index_file = MdFile(file_path=os.path.join(dir, "_index.md"), frontmatter_type=frontmatter_type)
    if not os.path.exists(index_file.file_path):
      index_file.dump_to_file(metadata={}, content="", dry_run=dry_run)
      metadata_helper.set_title_from_filename(index_file, transliteration_target=transliteration_target, dry_run=dry_run)
    elif overwrite:
      metadata_helper.set_title_from_filename(index_file, transliteration_target=transliteration_target, dry_run=dry_run)

def get_parent_md(md_file):
  if os.path.basename(md_file.file_path) == "_index.md":
    parent_dir = os.path.dirname(os.path.dirname(md_file.file_path))
  else:
    parent_dir = os.path.dirname(md_file.file_path)
  file_path = os.path.join(parent_dir, "_index.md")
  if os.path.exists(file_path):
    return MdFile(file_path=file_path)
  else:
    return None

def migrate_and_include(files, location_computer, new_url_computer, dry_run=False):
  """Migrate contents of a given file to a new location and include it in the original file"""
  logging.info("Processing %d files", len(files))
  migrate(files=files, location_computer=location_computer)
  for f in files:
    md_file = MdFile(file_path=f)
    (metadata, _) = md_file.read()
    md = """<div class="js_include" url="%s"  newLevelForH1="1" includeTitle="true"> </div>""" % new_url_computer(str(f))
    logging.info("Inclusion in old file : %s", md)
    md_file.dump_to_file(metadata=metadata, content=md, dry_run=dry_run)


# Autorzy: Sebastian Browarczyk, Filip Krzysztofik
# Silnik rekomendacji filmow

import json

import numpy as np

def euclidean_score(dataset, first_user, second_user):
    """Metoda służy do wyliczania podbieństwa pomiędzy użytkownikami
    Parametry wejściowe:
        dataset (dict): Zaimportowane dane użytkowników
        user1 (str): Nazwa pierwszego użytkownika
        user2 (str): Nazwa drugiego użytkownika
    Parametry zwracane:
        float: Stopień podobieństwa
    """
    if first_user not in dataset:
        raise TypeError('Nie można znaleść użytkownika :' + first_user)

    if second_user not in dataset:
        raise TypeError('Nie można znaleść użytkownika :' + second_user)

    common_movies = {}

    for item in dataset[first_user]:
        if item in dataset[second_user]:
            common_movies[item] = 1

    if len(common_movies) == 0:
        return 0

    squared_diff = []

    for item in dataset[first_user]:
        if item in dataset[second_user]:
            squared_diff.append(np.square(dataset[first_user][item] - dataset[second_user][item]))

    return 1 / (1 + np.sqrt(np.sum(squared_diff)))

def pearson_score(dataset, first_user, second_user):
    """Metoda służy do wyliczenie podonieństwa między użytkownika
    Parametry wejściowe:
        dataset (dict): zaimportowane dane użytkowników
        first_user (str): Nazwa pierwszego użytkownik
        second_user (str): Nazwa drugiego użytkownik
    Parametry zwracane:
            float: Stopień podobieństwa
    """
    if first_user not in dataset:
        raise TypeError('Nie można znaleść użytkownika :' + first_user)

    if second_user not in dataset:
        raise TypeError('Nie można znaleść użytkownika :' + second_user)

    common_movies = {}

    for item in dataset[first_user]:
        if item in dataset[second_user]:
            common_movies[item] = 1

    num_ratings = len(common_movies)

    if num_ratings == 0:
        return 0

    first_user_sum = np.sum([dataset[first_user][item] for item in common_movies])
    second_user_sum = np.sum([dataset[second_user][item] for item in common_movies])

    first_user_squared_sum = np.sum([np.square(dataset[first_user][item]) for item in common_movies])
    second_user_squared_sum = np.sum([np.square(dataset[second_user][item]) for item in common_movies])

    sum_of_products = np.sum([dataset[first_user][item] * dataset[second_user][item] for item in common_movies])

    Sxy = sum_of_products - (first_user_sum * second_user_sum / num_ratings)
    Sxx = first_user_squared_sum - np.square(first_user_sum) / num_ratings
    Syy = second_user_squared_sum - np.square(second_user_sum) / num_ratings

    if Sxx * Syy == 0:
        return 0

    return Sxy / np.sqrt(Sxx * Syy)

def print_result(user_name, user_dict):
    """Metoda służy do wyświtlania najlepiej i najgorzej ocenionych filmów uzytkownika
    Parametry wejściowe:
        user_name (str): Nazwa użytkownika
    Parametry zwracane:
        array[int]: Lista wyświetlonych filmów
    """
    print('Wybrano ' + user_name)
    movie_without_duplicates = {}
    for key, value in user_dict.items():
        if key not in data[user].keys():
            movie_without_duplicates[key] = value
    sorted_dict = dict(sorted(movie_without_duplicates.items(), key=lambda element: element[1], reverse=True))
    best7 = dict(list(sorted_dict.items())[:7])
    worse7 = dict(list(sorted_dict.items())[-7:])
    counter = 0
    movie_all = []

    print('\nPolecane filmy:')
    for itm in best7:
        print('[' + str(counter) + '] ' + itm + ' ' + str(best7[itm]))
        counter += 1
        movie_all.append(itm)

    print('\nNie polecane filmy:')
    for itm in worse7:
        print('[' + str(counter) + '] ' + itm + ' ' + str(worse7[itm]))
        counter += 1
        movie_all.append(itm)
    return movie_all

def select_user():
    """Metoda służy do wyboru użytkownika z listy
    Parametry zwracany:
        str:  Nazwa wybranego użytkownik
    """
    counter = 0
    user_list = []
    print('\nWybierz osobe:')
    for item in data:
        print('[' + str(counter) + '] ' + item)
        user_list.append(item)
        counter += 1
    return user_list[int(input())]

def select_score_type():
    """Metoda do wyświetlania możliwysz algorytmów i wyboru jednego z nich
    Parametry zwracany:
        str: Nazwa wybranego algorytmu
    """
    counter = 0
    score_type_dict = ['Pearson', 'Euclidean']
    score_type_list = []
    print('Wybierz algorytm')
    for item in score_type_dict:
        print('[' + str(counter) + '] ' + item)
        score_type_list.append(item)
        counter += 1
    return score_type_list[int(input())]

if __name__ == '__main__':
    json_file = 'movies.json'
    with open(json_file, 'r', encoding='utf-8') as file:
        data = json.loads(file.read())

    user = select_user()
    score_type = select_score_type()
    show_description = True
    score_array = []

    for item in data:
        if item != user:
            if score_type == 'Euclidean':
                score_array.append({'score': euclidean_score(data, user, item), 'user': item})
            else:
                score_array.append({'score': pearson_score(data, user, item), 'user': item})

    max_score = max(score_array, key=lambda x: x['score'])

    user_movies = data[max_score['user']]
    movie_all_list = print_result(max_score['user'], user_movies)


from pathlib import Path
from typing import Union

import numpy as np
import pandas as pd
import eccodes

import dask
from dask.distributed import Client, progress

from reki.data_finder import find_local_file
from reki.format.grib.eccodes import load_message_from_file
from reki_data_tool.utils import cal_run_time
from reki_data_tool.ml.southwest_china.data_proc.common import get_meso_parameters

from reki_data_tool.ml.southwest_china.config import OUTPUT_BASE_DIRECTORY

def create_serial(
        start_time: pd.Timestamp,
        forecast_time: pd.Timedelta,
        output_file_path: Union[Path, str]
):
    parameters = get_meso_parameters()

    file_path = find_local_file(
        "grapes_meso_3km/grib2/orig",
        start_time=start_time,
        forecast_time=forecast_time
    )

    with open(output_file_path, "wb") as f:
        for p in parameters:
            m = load_message_from_file(file_path, **p)
            if m is None:
                continue

            b = eccodes.codes_get_message(m)
            eccodes.codes_release(m)
            f.write(b)
    return True

def main():
    date_list = pd.date_range(
        "2022-06-01 00:00", "2022-06-30 23:59",
        freq="12h",
    )
    forecast_hours = np.arange(0, 37, 6)

    client = Client(
        threads_per_worker=1,
        n_workers=32,
    )
    print(client)

    tasks = []
    for date in date_list:
        for hour in forecast_hours:
            start_time = date
            start_time_label = start_time.strftime("%Y%m%d%H")
            forecast_time = pd.Timedelta(hours=hour)
            forecast_time_label = f"{int(forecast_time / pd.Timedelta(hours=1)):03}"

            output_file_name = f"meso_cn_{start_time_label}_{forecast_time_label}.grb2"
            output_file_path = Path(OUTPUT_BASE_DIRECTORY, "meso", output_file_name)

            t = dask.delayed(create_serial)(start_time, forecast_time, output_file_path)
            tasks.append(t)

    def get_tasks(tasks):
        return tasks

    t = dask.delayed(get_tasks)(tasks)

    result = t.persist()

    progress(result)

    r = result.compute()

    client.close()

if __name__ == "__main__":
    main()


import numpy as np
import pandas as pd
import matplotlib.pyplot as py
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
import pickle

df_wine=pd.read_csv('C:\DataScience\Machine Learning\Iinear_regression\QualityPrediction.csv')
df_wine
# print(str(df_wine.shape))
# df_wine['quality'].fillna(0,inplace=True)
badindex=df_wine.loc[df_wine['quality'] <= 5].index
goodindex=df_wine.loc[df_wine['quality'] > 5].index
df_wine.iloc[badindex, df_wine.columns.get_loc('quality')]=0
df_wine.iloc[goodindex, df_wine.columns.get_loc('quality')]=1
x=df_wine.drop('quality',axis=1)
y=df_wine['quality']
# Splitting Training and Test Set
# choosing 20% as  training data
xTrain,xTest,yTrain,yTest=train_test_split(x,y,test_size=0.20,random_state=42)
regressor=LinearRegression()

# Fitting model with training data
regressor.fit(xTrain,yTrain)
# y_prediction_lr=regressor.predict(xTest)

# Saving model to disk
pickle.dump(regressor,open('model.pkl','wb'))

# Loading model to compare the results
model=pickle.load(open('model.pkl','rb'))

# print(model.predict([[7.9,0.35,0.46,3.6,0.078,15,37,0.9973,3.35,0.86,12.8]]))
print(model.predict([[7.4,0.7,0,1.9,0.076,11,34,0.9978,3.51,0.56,9.4]]))


import os

from invoke import task
from .utils import *

@task("homebrew")
def ack():
  brew_install("ack")

@task("homebrew")
def autojump():
  if brew_install("autojump"):
    print (
      """
      The following needs to be added to your .bash_profile or .zshrc,

      [[ -s `brew --prefix`/etc/autojump.sh ]] && . `brew --prefix`/etc/autojump.sh
      """
    )

@task("homebrew")
def coreutils():
  brew_install("coreutils")

@task("homebrew")
def ctags():
  brew_install("ctags")

@task("virtualbox")
def docker():
  brew_install("boot2docker")
  brew_install("docker")

@task("homebrew")
def htop():
  brew_install("htop-osx")

@task("python")
def httpie():
  if command_exists("http"): return
  pip_install("httpie")

@task("homebrew")
def jq():
  brew_install("jq")

@task("homebrew")
def lua():
  brew_install("lua")

@task("lua")
def luarocks():
  brew_install("luarocks")

@task("homebrew")
def mysql():
  HOME = os.environ["HOME"]
  brew_install("mysql")

  # register mysql with Launch Control to start on system startup
  source = '/usr/local/opt/mysql/homebrew.mxcl.mysql.plist'
  dest   = '{}/Library/LaunchAgents/homebrew.mxcl.mysql.plist'.format(HOME)
  if not os.path.exists(dest):
    run('ln -sfv {source} {dest}'.format(source=source, dest=dest))
  if 'homebrew.mxcl.mysql' not in run("launchctl list", hide="out").stdout:
    run('launchctl load {dest}'.format(dest))

@task("homebrew")
def parallel():
  brew_install("parallel")

@task("homebrew")
def python():
  brew_install("python")

@task("python")
def python_scientific():
  # scipy
  brew_install("gcc")

  # matplotlib
  brew_install("freetype")
  pip_install("pyparsing")

  # `pip install scipy` doesn't work with Homebrew due to path issues, so let
  # homebrew handle its installation instead.
  brew_tap("samueljohn/python")
  brew_install("numpy")
  brew_install("scipy")
  brew_install("matplotlib")

  pip_install(["pandas", "ipython", "ipdb"])

@task("python")
def python_productivity():
  brew_install("libevent")
  brew_install( "libyaml")
  brew_install( "libxml2")
  pip_install(         "PyYAML")
  pip_install("python-dateutil")
  pip_install([
    "configurati",
    "duxlib",
    "funcy",
    "gevent",
    "invoke",
    "lxml",
    "virtualenv",
  ])

@task("python", "mysql")
def python_web():
  pip_install("MySQL-python")
  pip_install([
    "sqlrest",
    "bottle",
    "pyjade",
    "jinja2",
  ])

@task("python")
def python_testing():
  pip_install([
    "nose",
  ])

@task("python")
def python_amazon():
  pip_install([
    "boto",
    "awscli",
  ])
  pip_install("gsutil")

@task("homebrew")
def s3cmd():
  brew_install("s3cmd")

@task("homebrew")
def scala():
  brew_install("scala")

@task("homebrew", "dotfiles")
def tmux():
  brew_install("tmux")
  brew_install("reattach-to-user-namespace")

@task("homebrew")
def tree():
  brew_install("tree")

@task("homebrew")
def wget():
  brew_install("wget")

@task("homebrew", "dotfiles")
def zsh():
  if command_exists("zsh"): return
  brew_install("zsh")
  run('chsh -s $(which zsh)')


import os
from PIL import Image
import datetime
import json
import getArea

def create_image_info(
    image_id,
    file_name,
    image_size,
    date_captured=datetime.datetime.utcnow().isoformat(" "),
    license_id=1,
    coco_url="",
    flickr_url="",
):
    image_info = {
        "id": image_id,
        "file_name": file_name,
        "width": image_size[0],
        "height": image_size[1],
        "date_captured": date_captured,
        "license": license_id,
        "coco_url": coco_url,
        "flickr_url": flickr_url,
    }

    return image_info

def create_annotation_info(
    annotation_id, image_id, category_id, is_crowd, area, bounding_box, segmentation
):
    annotation_info = {
        "id": annotation_id,
        "image_id": image_id,
        "category_id": category_id,
        "iscrowd": is_crowd,
        "area": area,  # float
        "bbox": bounding_box,  # [x,y,width,height]
        "segmentation": segmentation,  # [polygon]
    }

    return annotation_info

def get_segmenation(coord_x, coord_y):
    seg = []
    for x, y in zip(coord_x, coord_y):
        seg.append(x)
        seg.append(y)
    return [seg]

def convert(
    imgdir,
    annpath,
    categories=None,
    super_categories=None,
    output_file_name=None,
    first_class_index=1,  # typically, 0 or 1
):
    """
    :param imgdir: directory for your images
    :param annpath: path for your annotations
    :return: coco_output is a dictionary of coco style which you could dump it into a json file
    as for keywords 'info','licenses','categories',you should modify them manually
    """

    if categories is None:
        categories = ["rib", "clavicle"]

    default_category = categories[0]

    category_dict = dict()
    for (cat_id, cat_name) in enumerate(categories, start=first_class_index):
        category_dict[cat_name] = cat_id

    if super_categories is None:
        default_super_category = "bone"
        super_categories = [default_super_category for _ in categories]

    coco_output = {}
    coco_output["info"] = {
        "description": "Example Dataset",
        "url": "https://github.com/waspinator/pycococreator",
        "version": "0.1.0",
        "year": 2018,
        "contributor": "waspinator",
        "date_created": datetime.datetime.utcnow().isoformat(" "),
    }
    coco_output["licenses"] = [
        {
            "id": 1,
            "name": "Attribution-NonCommercial-ShareAlike License",
            "url": "http://creativecommons.org/licenses/by-nc-sa/2.0/",
        }
    ]
    coco_output["categories"] = [
        {
            "id": category_dict[cat_name],
            "name": cat_name,
            "supercategory": super_cat_name,
        }
        for (cat_name, super_cat_name) in zip(categories, super_categories)
    ]
    coco_output["images"] = []
    coco_output["annotations"] = []

    ann = json.load(open(annpath))
    # annotations id start from zero
    ann_id = 0
    # in VIA annotations, keys are image name
    for img_id, key in enumerate(ann.keys()):

        filename = ann[key]["filename"]
        img = Image.open(imgdir + filename)
        # make image info and storage it in coco_output['images']
        image_info = create_image_info(
            img_id, os.path.basename(filename), image_size=img.size
        )
        # Caveat: image shapes are conventionally (height, width) whereas image sizes are conventionally (width, height)
        # References:
        # -   https://note.nkmk.me/en/python-opencv-pillow-image-size/
        # -   https://github.com/facebookresearch/detectron2/blob/master/detectron2/data/detection_utils.py#L189
        coco_output["images"].append(image_info)
        regions = ann[key]["regions"]
        # for one image ,there are many regions,they share the same img id
        for region in regions:
            region_attributes = regions[region]["region_attributes"]
            try:
                cat_name = region_attributes["label"]
            except KeyError:
                cat_name = default_category
            try:
                cat_id = category_dict[cat_name]
            except KeyError:
                print("Skipping unknown category {} in {}".format(cat_name, filename))
                continue
            iscrowd = 0
            shape_attributes = regions[region]["shape_attributes"]
            points_x = shape_attributes["all_points_x"]
            points_y = shape_attributes["all_points_y"]
            area = getArea.GetAreaOfPolyGon(points_x, points_y)
            min_x = min(points_x)
            max_x = max(points_x)
            min_y = min(points_y)
            max_y = max(points_y)
            box = [min_x, min_y, max_x - min_x, max_y - min_y]
            segmentation = get_segmenation(points_x, points_y)
            # make annotations info and storage it in coco_output['annotations']
            ann_info = create_annotation_info(
                ann_id, img_id, cat_id, iscrowd, area, box, segmentation
            )
            coco_output["annotations"].append(ann_info)
            ann_id = ann_id + 1

    if output_file_name is not None:
        print("Saving to {}".format(output_file_name))

        with open(output_file_name, "w") as f:
            json.dump(coco_output, f)

    return coco_output

if __name__ == "__main__":
    input_dir = "/content/data/balloon/train/"
    input_json = input_dir + "via_region_data.json"
    categories = ["balloon"]

    super_categories = ["N/A"]

    output_json = input_dir + "coco_train.json"

    coco_dict = convert(
        imgdir=input_dir,
        annpath=input_json,
        categories=categories,
        super_categories=super_categories,
        output_file_name=output_json,
    )


r"""
Global Correction
-----------------

Iterative Correction Scheme.

The response matrices :math:`R_{O}` for the observables :math:`O` (e.g. BBX, PHASEX, ...)
are loaded from a file and then the equation

.. math:: R_{O} \cdot \delta var = O_{meas} - O_{model}
    :label: eq1

is being solved for :math:`\delta var` via a chosen method (at the moment only numpys pinv,
which creates a pseudo-inverse via svd is used).

The response matrices are hereby merged into one matrix for all observables to solve for all
:math:`\delta var` at the same time.

To normalize the observables to another ``weigths`` (W) can be applied.

Furthermore, an ``errorcut``, specifying the maximum errorbar for a BPM to be used, and
``modelcut``, specifying the maximum distance between measurement and model for a BPM to be used,
can be defined. Data from BPMs outside of those cut-values will be discarded.
These cuts are defined for each observable separately.

After each iteration the model variables are changed by :math:`-\delta var` and the
observables are recalculated by Mad-X.
:eq:`eq1` is then solved again.

Input arguments are split into correction arguments and accelerator arguments.
The former are listed below, the latter depend on the accelerator you want
to use. Check :ref:`modules/model:Model` to see which ones are needed.

**Arguments:**

*--Required--*

- **meas_dir**:

    Path to the directory containing the measurement files.

- **output_dir**:

    Path to the directory where to write the output files.

*--Optional--*

- **beta_filename**:

    Prefix of the beta file to use. E.g.: getkmodbeta

    default: ``beta_phase_``

- **errorcut** *(float)*:

    Reject BPMs whose error bar is higher than the corresponding input.
    Input in order of optics_params.

- **fullresponse_path**:

    Path to the fullresponse binary file.If not given, calculates the
    response analytically.

- **max_iter** *(int)*:

    Maximum number of correction re-iterations to perform. A value of `0`
    means the correction is calculated once.

    default: ``3``

- **method** *(str)*:

    Optimization method to use.

    choices: ``('pinv', 'omp')``

    default: ``pinv``

- **min_corrector_strength** *(float)*:

    Minimum (absolute) strength of correctors.

    default: ``0.0``

- **modelcut** *(float)*:

    Reject BPMs whose deviation to the model is higher than the
    corresponding input. Input in order of optics_params.

- **n_correctors** *(int)*:

    Maximum number of correctors to use. (Method: 'omp')

- **optics_params** *(str)*:

    List of parameters to correct upon (e.g. BETX BETY)

    choices: ``('PHASEX', 'PHASEY', 'BETX', 'BETY', 'DX', 'DY', 'NDX', 'Q', 'F1001R', 'F1001I', 'F1010R', 'F1010I')``

    default: ``['PHASEX', 'PHASEY', 'BETX', 'BETY', 'NDX', 'Q']``

- **output_filename**:

    Identifier of the output files.

    default: ``changeparameters_iter``

- **svd_cut** *(float)*:

    Cutoff for small singular values of the pseudo inverse. (Method:
    'pinv')Singular values smaller than rcond*largest_singular_value are
    set to zero

    default: ``0.01``

- **update_response**:

    Update the (analytical) response per iteration.

    action: ``store_true``

- **use_errorbars**:

    Take into account the measured errorbars in the correction.

    action: ``store_true``

- **variable_categories**:

    List of names of the variables classes to use.

    default: ``['MQM', 'MQT', 'MQTL', 'MQY']``

- **weights** *(float)*:

    Weight to apply to each measured quantity. Input in order of
    optics_params.

Possible problems and notes (lmalina, 2020):
 * error-based weights default? likely - but be careful with low tune errors vs svd cut in pseudoinverse
 * manual creation of pd.DataFrame varslist, deltas? maybe tunes in tfs_pandas single value or a column?
 * There should be some summation/renaming for iterations
 * For two beam correction
 * The two beams can be treated separately until the calculation of correction
 * Values missing in the response (i.e. correctors of the other beam) shall be treated as zeros
 * Missing a part that treats the output from LSA

"""
from pathlib import Path
from typing import Dict

from generic_parser import DotDict
from generic_parser.entrypoint_parser import EntryPointParameters, entrypoint
from omc3.correction import handler
from omc3.optics_measurements.constants import (BETA, DISPERSION, F1001, F1010,
                                                NORM_DISPERSION, PHASE, TUNE)
from omc3.model import manager
from omc3.utils import logging_tools
from omc3.utils.iotools import PathOrStr, save_config

LOG = logging_tools.get_logger(__name__)

OPTICS_PARAMS_CHOICES = (f"{PHASE}X", f"{PHASE}Y",
                         f"{BETA}X", f"{BETA}Y",
                         f"{NORM_DISPERSION}X",
                         f"{TUNE}",
                         f"{DISPERSION}X", f"{DISPERSION}Y",
                         f"{F1001}R", f"{F1001}I", f"{F1010}R", f"{F1010}I")

CORRECTION_DEFAULTS = {
    "optics_file": None,
    "output_filename": "changeparameters_iter",
    "svd_cut": 0.01,
    "optics_params": OPTICS_PARAMS_CHOICES[:6],
    "variable_categories": ["MQM", "MQT", "MQTL", "MQY"],
    "beta_filename": "beta_phase_",
    "method": "pinv",
    "iterations": 4,
}

def correction_params():
    params = EntryPointParameters()
    params.add_parameter(name="meas_dir",
                         required=True,
                         type=PathOrStr,
                         help="Path to the directory containing the measurement files.",)
    params.add_parameter(name="output_dir",
                         required=True,
                         type=PathOrStr,
                         help="Path to the directory where to write the output files.", )
    params.add_parameter(name="fullresponse_path",
                         type=PathOrStr,
                         help="Path to the fullresponse binary file.If not given, "
                              "calculates the response analytically.",)
    params.add_parameter(name="optics_params",
                         type=str,
                         nargs="+",
                         default=list(CORRECTION_DEFAULTS["optics_params"]),
                         choices=OPTICS_PARAMS_CHOICES,
                         help=f"List of parameters to correct upon (e.g. {BETA}X {BETA}Y)", )
    params.add_parameter(name="output_filename",
                         default=CORRECTION_DEFAULTS["output_filename"],
                         help="Identifier of the output files.", )
    params.add_parameter(name="min_corrector_strength",
                         type=float,
                         default=0.,
                         help="Minimum (absolute) strength of correctors.",)
    params.add_parameter(name="modelcut",
                         nargs="+",
                         type=float,
                         help="Reject BPMs whose deviation to the model is higher "
                              "than the corresponding input. Input in order of optics_params.",)
    params.add_parameter(name="errorcut",
                         nargs="+",
                         type=float,
                         help="Reject BPMs whose error bar is higher than the corresponding "
                              "input. Input in order of optics_params.",)
    params.add_parameter(name="weights",
                         nargs="+", type=float,
                         help="Weight to apply to each measured quantity. "
                              "Input in order of optics_params.",)
    params.add_parameter(name="variable_categories",
                         nargs="+",
                         default=CORRECTION_DEFAULTS["variable_categories"],
                         help="List of names of the variables classes to use.", )
    params.add_parameter(name="beta_filename",
                         default=CORRECTION_DEFAULTS["beta_filename"],
                         help="Prefix of the beta file to use. E.g.: beta_phase_", )
    params.add_parameter(name="method",
                         type=str,
                         choices=("pinv", "omp"),
                         default=CORRECTION_DEFAULTS["method"],
                         help="Optimization method to use.", )
    params.add_parameter(name="svd_cut",
                         type=float,
                         default=CORRECTION_DEFAULTS["svd_cut"],
                         help="Cutoff for small singular values of the pseudo inverse. "
                              "(Method: 'pinv')Singular values smaller than "
                              "rcond*largest_singular_value are set to zero", )
    params.add_parameter(name="n_correctors",
                         type=int,
                         help="Maximum number of correctors to use. (Method: 'omp')")
    params.add_parameter(name="iterations",
                         type=int,
                         default=CORRECTION_DEFAULTS["iterations"],
                         help="Maximum number of correction iterations to perform. "
                              "A value of `1` means the correction is calculated once."
                              "In this case, the accelerator instance does not need to be able"
                              "to produce a new model.", )
    params.add_parameter(name="use_errorbars",
                         action="store_true",
                         help="Take into account the measured errorbars as weights.", )
    params.add_parameter(name="update_response",
                         action="store_true",
                         help="Update the (analytical) response per iteration.", )
    return params

@entrypoint(correction_params())
def global_correction_entrypoint(opt: DotDict, accel_opt) -> None:
    """Do the global correction. Iteratively."""
    LOG.info("Starting Iterative Global Correction.")
    save_config(Path(opt.output_dir), opt, __file__, unknown_opt=accel_opt)

    opt = _check_opt_add_dicts(opt)
    opt = _add_hardcoded_paths(opt)
    opt.output_dir.mkdir(parents=True, exist_ok=True)
    accel_inst = manager.get_accelerator(accel_opt)
    handler.correct(accel_inst, opt)

def _check_opt_add_dicts(opt: dict) -> dict:  # acts inplace...
    """ Check on options and put in missing values """
    def_dict = _get_default_values()

    # Check cuts and fill defaults
    for key in ("modelcut", "errorcut", "weights"):
        if opt[key] is None:
            opt[key] = [def_dict[key][p] for p in opt.optics_params]
        elif len(opt[key]) != len(opt.optics_params):
            raise AttributeError(f"Length of {key} is not the same as of the optical parameters!")
        opt[key] = dict(zip(opt.optics_params, opt[key]))

    # Convert Strings to Paths
    opt.meas_dir = Path(opt.meas_dir)
    opt.output_dir = Path(opt.output_dir)
    if opt.fullresponse_path:
        opt.fullresponse_path = Path(opt.fullresponse_path)
    return opt

def _add_hardcoded_paths(opt: DotDict) -> DotDict:  # acts inplace...
    opt.change_params_path = opt.output_dir / f"{opt.output_filename}.madx"
    opt.change_params_correct_path = opt.output_dir / f"{opt.output_filename}_correct.madx"
    opt.knob_path = opt.output_dir / f"{opt.output_filename}.tfs"
    return opt

# Define functions here, to new optics params
def _get_default_values() -> Dict[str, Dict[str, float]]:
    return {
        "modelcut": {
            f"{PHASE}X": 0.05,
            f"{PHASE}Y": 0.05,
            f"{BETA}X": 0.2,
            f"{BETA}Y": 0.2,
            f"{DISPERSION}X": 0.2,
            f"{DISPERSION}Y": 0.2,
            f"{NORM_DISPERSION}X": 0.2,
            f"{TUNE}": 0.1,
            f"{F1001}R": 0.2,
            f"{F1001}I": 0.2,
            f"{F1010}R": 0.2,
            f"{F1010}I": 0.2,
        },
        "errorcut": {
            f"{PHASE}X": 0.035,
            f"{PHASE}Y": 0.035,
            f"{BETA}X": 0.02,
            f"{BETA}Y": 0.02,
            f"{DISPERSION}X": 0.02,
            f"{DISPERSION}Y": 0.02,
            f"{NORM_DISPERSION}X": 0.02,
            f"{TUNE}": 0.027,
            f"{F1001}R": 0.02,
            f"{F1001}I": 0.02,
            f"{F1010}R": 0.02,
            f"{F1010}I": 0.02,
        },
        "weights": {
            f"{PHASE}X": 1,
            f"{PHASE}Y": 1,
            f"{BETA}X": 0,
            f"{BETA}Y": 0,
            f"{DISPERSION}X": 0,
            f"{DISPERSION}Y": 0,
            f"{NORM_DISPERSION}X": 0,
            f"{TUNE}": 10,
            f"{F1001}R": 0,
            f"{F1001}I": 0,
            f"{F1010}R": 0,
            f"{F1010}I": 0,
        },
    }

if __name__ == "__main__":
    global_correction_entrypoint()


import numpy as np
from matplotlib import pyplot as plt

N = 100
x = np.linspace(0, 1 / 8, N)
y = 3 * x * np.sin(x)

# a
tkeo = []
for i in range(1, N - 1):
    tkeo.append(y[i] ** 2 - y[i + 1] * y[i - 1])

print(tkeo)

# b
s = 0

sum_mian = 0
for e in range(N):
    sum_mian += (y[e] - sum(y) / len(y)) ** 3
mianownik = 1/N * sum_mian

sum_licz = 1/N
for e in range(N):
    sum_licz += (y[e] - sum(y) / len(y))

s = 1/(N-1) * sum_licz ** (3/2)
print(s)

# c

plt.subplot(211)
plt.plot(x[1:-1], tkeo, 'y-')

plt.subplot(212)
plt.plot(x, y, 'y-')

plt.show()


from xai.brain.wordbase.nouns._mecca import _MECCA

#calss header
class _MECCAS(_MECCA, ):
	def __init__(self,):
		_MECCA.__init__(self)
		self.name = "MECCAS"
		self.specie = 'nouns'
		self.basic = "mecca"
		self.jsondata = {}


from random import shuffle
import collections
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cross_validation import cross_val_score
from sklearn import svm
from sklearn.metrics import f1_score,precision_score,recall_score
from text_preprocessing.DatasetParser import Cleaning
from nltk.tokenize import sent_tokenize
def get_data(positive_polarity_filename,negative_polarity_filename):
    reviews = []
    labels = []
    vocab = collections.defaultdict(float)
    with open(positive_polarity_filename,"r",encoding='utf-8') as pos_file:
        data = pos_file.readlines()
        for line in data:
            review = (line.strip("0123456789").lower())
            words = set(review.split())
            for word in words:
                vocab[word] += 1
            reviews.append(review)
            labels.append(1)

    with open(negative_polarity_filename,"r",encoding='utf-8') as neg_file:
        data = neg_file.readlines()
        for line in data:
            review = (line.strip("0123456789").lower())
            words = set(review.split())
            for word in words:
                vocab[word] += 1
            reviews.append(review)
            labels.append(0)
    # Generate counts from text using a vectorizer.  There are other vectorizers available, and lots of options you can set.
    # This performs our step of computing word counts
    data = []
    for rev,label in zip(reviews,labels):
        data.append([rev,label])
    shuffle(data)
    l = len(data) - len(data)//10
    # print(l)
    train = data[:l]
    test = data[(l+1):len(data)]
    print("training samples - ",len(train),"test samples - ",len(test))
    return train,test
    # return reviews,labels,vocab
    # return data

def multinomial_naive_bayes(train_features,test_features,f1_dataset_features,y_true):
    nb = MultinomialNB()
    cv_score = cross_val_score(nb,train_features,[r[1] for r in train],cv=10)
    print("cv score: ",cv_score.mean())

    nb.fit(train_features, [int(r[1]) for r in train])
    y_pred = nb.predict(f1_dataset_features)
    print("f1-score: ",f1_score(y_true,y_pred,average='binary'))
    print("precision: ",precision_score(y_true,y_pred,average='binary'))
    print("recall: ",recall_score(y_true,y_pred,average='binary'))
    # actual = [int(r[1]) for r in test]
    # Now we can use the model to predict classifications for our test features.
    #predictions = nb.predict(test_features)
    #print(nb.predict(vectorizer.transform(["below par food which is salty and oily ."])))
    # Compute the error.  It is slightly different from our model because the internals of this process work differently from our implementation.
    # fpr, tpr, thresholds = metrics.roc_curve(actual, predictions, pos_label=1)
    # print("Multinomial naive bayes AUC: {0}".format(metrics.auc(fpr, tpr)))

def svm_linear(train_features,test_features,f1_dataset_features,y_true):
    print("Training SVM linear")
    svc = svm.SVC(kernel="linear")
    # svc = GridSearchCV(svc,cv=10)
    cv_score = cross_val_score(svc,train_features,[r[1] for r in train],cv=10)
    print("cv score: ",cv_score.mean())
    svc = svc.fit(train_features,[int(r[1]) for r in train])
    y_pred = svc.predict(f1_dataset_features)
    print("f1-score: ",f1_score(y_true,y_pred,average='binary'))
    print("precision: ",precision_score(y_true,y_pred,average='binary'))
    print("recall: ",recall_score(y_true,y_pred,average='binary'))
    # print(svc.predict(vectorizer.transform(["below par food which is salty and oily ."])))

def svm_rbf(train_features,test_features,f1_dataset_features,y_true):
    print("Training SVM rbf")
    svc = svm.SVC(kernel="rbf",gamma=0.7)
    # svc = GridSearchCV(svc,cv=10)
    cv_score = cross_val_score(svc,train_features,[r[1] for r in train],cv=10)
    print(cv_score.mean())
    svc = svc.fit(train_features,[int(r[1]) for r in train])
    y_pred = svc.predict(f1_dataset_features)
    print("f1-score: ",f1_score(y_true,y_pred,average='binary'))
    print("precision: ",precision_score(y_true,y_pred,average='binary'))
    print("recall: ",recall_score(y_true,y_pred,average='binary'))
    # print(svc.predict(vectorizer.transform(["below par food which is salty and oily ."])))

def svm_polynomial(train_features,test_features,f1_dataset_features,y_true):
    print("Training SVM polynomial")
    svc = svm.SVC(kernel="poly")
    # svc = GridSearchCV(svc,cv=10)
    cv_score = cross_val_score(svc,train_features,[r[1] for r in train],cv=10)
    print(cv_score.mean())
    svc = svc.fit(train_features,[int(r[1]) for r in train])
    y_pred = svc.predict(f1_dataset_features)
    print("f1-score: ",f1_score(y_true,y_pred,average='binary'))
    print("precision: ",precision_score(y_true,y_pred,average='binary'))
    print("recall: ",recall_score(y_true,y_pred,average='binary'))
    # print(svc.predict(vectorizer.transform(["below par food which is salty and oily ."])))

def svm_sigmoid(train_features,test_features,f1_dataset_features,y_true):
    print("Training SVM sigmoid")
    svc = svm.SVC(kernel="sigmoid")
    # svc = GridSearchCV(svc,cv=10)
    cv_score = cross_val_score(svc,train_features,[r[1] for r in train],cv=10)
    print(cv_score.mean())
    svc = svc.fit(train_features,[int(r[1]) for r in train])
    y_pred = svc.predict(f1_dataset_features)
    print("f1-score: ",f1_score(y_true,y_pred,average='binary'))
    print("precision: ",precision_score(y_true,y_pred,average='binary'))
    print("recall: ",recall_score(y_true,y_pred,average='binary'))
    # print(svc.predict(vectorizer.transform(["below par food which is salty and oily ."])))
def get_f1_dataset():
    _cleansing = Cleaning()
    pos = _cleansing.get_clean_data("../../../../data/f1_dataset/pos")
    neg = _cleansing.get_clean_data("../../../../data/f1_dataset/neg")
    pos = sent_tokenize(pos)
    neg = sent_tokenize(neg)
    return pos,neg
if __name__ =="__main__":
    vectorizer = TfidfVectorizer(stop_words='english',min_df=1)
    train,test = get_data("data/pos.txt","data/neg.txt")
    pos_f1,neg_f1 = get_f1_dataset()
    f1_dataset = pos_f1+neg_f1
    y_true = [1]*len(pos_f1) + [0]*len(neg_f1)
    train_features = vectorizer.fit_transform([r[0] for r in train])
    test_features = vectorizer.transform([r[0] for r in test])
    f1_dataset_features= vectorizer.transform([pos for pos in f1_dataset])

    multinomial_naive_bayes(train_features,test_features,f1_dataset_features,y_true)
    svm_linear(train_features,test_features,f1_dataset_features,y_true)
    svm_rbf(train_features,test_features,f1_dataset_features,y_true)


# -*- coding: utf-8 -*-

"""
@Author: lemons
@Date:
@Description: 封装篇章结构体
"""

class Document:
    def __init__(self):
        # 针对内容
        self.text_word_ids = ...
        self.text_pos_ids = ...
        self.text_ent_ids = ...
        # 针对标题
        self.title_word_ids = ...
        self.title_pos_ids = ...
        self.title_ent_ids = ...


# BACKJOON #2156 <포도주 시식>
# https://www.acmicpc.net/problem/2156

a = [ int(input()) for _ in ' '*int(input()) ]
if len(a) < 3: print(sum(a))
else:
    m1,m2,m3=0,a[0],a[0]+a[1]
    for i in range(2,len(a)):
        m1,m2,m3=m2,m3,max(m1+a[i-1]+a[i],m2+a[i],m3)
    print(m3)


import linecache
import traceback
import resource
import filecmp
from sklearn.utils import shuffle
import tensorflow as tf
import subprocess
import json
import seaborn as sns
import matplotlib.pyplot as plt
import re
import hashlib
import numpy as np
import os,sys
import pickle
import gensim.models as w2v
import pandas as pd
import nltk
import generateFeatures as feat
from bert_serving.server.helper import get_args_parser
from bert_serving.server.helper import get_shutdown_parser
from bert_serving.server import BertServer
from bert_serving.client import BertClient
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.FATAL)

filename = sys.argv[0]
cwd = os.path.abspath(filename+"/..")

modelpath = cwd+"/w2vModel"
data_dir = cwd+"/data"
dataset_dir = data_dir+"/datasets"
bert_dir = cwd+"/res/bert"
hash_path = cwd+"/data/hash.txt"
spec_dir = cwd+"/res/specificity/Domain-Agnostic-Sentence-Specificity-Prediction"

#max num of words in a sentence
#only used for w2v
MAX_SENT_LEN = 2000
MIN_BODY_LEN = 50
EMB_DIM_SIZE = 300
#EMB_DIM_SIZE = 100

stages = ["concat","data","folds","bert","features","complexity","specificity","w2v"]

def clean_text(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip()

def pos(sentence):
    tokens = nltk.word_tokenize(sentence)
    tagsAndTokes = nltk.pos_tag(tokens)
    justTags = " ".join([e[1] for e in tagsAndTokes])
    return justTags

class LineSeekableFile:
    def __init__(self, seekable):
        self.c = 0
        self.fin = seekable
        self.line_map = list() # Map from line index -> file position.
        self.line_map.append(0)
        while seekable.readline():
            print("Creating LineSeekableFile Object: ",self.c)
            self.c += 1
            self.line_map.append(seekable.tell())

    def __getitem__(self, index):
        # NOTE: This assumes that you're not reading the file sequentially.
        # For that, just use 'for line in file'.
        self.fin.seek(self.line_map[index])
        return self.fin.readline()

def save_p(filename, data):
    with open(filename, "wb") as p:
        pickle.dump(data, p, protocol=4)

def read_p(filename):
    with open(filename, "rb") as f:
        return pickle.load(f)

#true if data is the same, False if not
def check_hash(df_hash, num_folds, drop_feat_idx=None, stage="data"):
    print(stage.upper(), " CHECK> Checking Hash Path on: ",hash_path)
    if os.path.exists(hash_path):
        with open(hash_path, "r") as h:
            lines = h.readlines()
            old_folds = lines[1].split(" ")[1][:-1]
            old_feat_idx = lines[-2].split("drop_feat: ")[1][:-1]
            print("old_feat_idx:", old_feat_idx)
            for i,l in enumerate(lines):
                if l.startswith(stage):
                    old_hash = l.split(" ")[1][:-1]

            print("Old and New Hash: ",old_hash[:5],df_hash[:5]," Same? ", (old_hash == df_hash))
            print("Old and New #folds:",old_folds, num_folds, " Same? ", (int(old_folds)==num_folds))
            print("Old and New Drop_Feat: ",old_feat_idx,drop_feat_idx," Same? ", (old_feat_idx == str(drop_feat_idx)),"\n")

        if old_hash != df_hash:
            return False
        if stage in ["drop_feat", "data"] and (old_feat_idx != str(drop_feat_idx)):
            return False
        if stage == "data" and (int(old_folds) != num_folds):
            return False
        return True
    return False

def savehash(stage, hashcode, drop_feat_idx=None):
    with open(hash_path, "r") as h:
        data = h.readlines()
        for i,l in enumerate(data):
            if l.startswith(stage):
                if stage == "drop_feat":
                    data[i] = stage+": "+str(drop_feat_idx)+"\n"
                else:
                    data[i] = stage+": "+hashcode+"\n"
    with open(hash_path, "w") as h:
        h.writelines(data)

def reset_hash(force_reload):
    if force_reload == "all" :
        for s in stages:
            savehash(s,"0")
    if force_reload == "emb" :
        for s in ["data", "bert", "concat"]:
            savehash(s,"0")
    if force_reload == "feat" :
        for s in ["data", "features", "concat", "complexity", "specificity"]:
            savehash(s,"0")
    if force_reload == "just_reload" :
        for s in ["data", "concat"]:
            savehash(s,"0")

#run concat+normalize.py inside dataset/ before loading data
def load_data(emb_type='w2v', collapse_classes=False, fold_test=None, num_folds=1, random_state=None, force_reload=None, drop_feat_idx=[], only_claims=False, feature_list=["inf","div","qua","aff","sbj","spe","pau","unc","pas"]):
    print('Loading data from',dataset_dir)
    data = pd.read_csv(dataset_dir+"/dataset.csv", sep=',')

    if force_reload is not None: reset_hash(force_reload)

    print("size of initial \"dataset\":",len(data))
    #determines if data will be whole body or only claims
    #df = df[lens < MAX_SENT_LEN]
    if only_claims:
        df = data[['claim','verdict']].copy()
        df = df.rename(columns={"claim": "body"})
        df = df[df['body'].map(len) > MIN_BODY_LEN]
        print("after dropping origins with less than "+str(MIN_BODY_LEN)+" chars:",len(df))
    else:
        df = data[['o_body','verdict']].copy()
        df = df.rename(columns={"o_body": "body"})
    if 'o_url' in df.columns:
        df = df.drop_duplicates(subset='o_url', keep='first')
    print("after dropping duplicates:",len(df))
    df.body = df.body.astype('str')
    df.verdict = df.verdict.astype('str')
    df['verdict'] = df['verdict'].str.lower()
    df.body.apply(clean_text)
    df = df.reset_index()

    if(collapse_classes):
        print("labels before collapse classes:", df.verdict.unique())
        df.loc[df['verdict'] == "mfalse", 'verdict'] = 'false'
        df.loc[df['verdict'] == "false.", 'verdict'] = 'false'
        df.loc[df['verdict'] == "mtrue", 'verdict'] = 'true'
        df.loc[df['verdict'] == "true.", 'verdict'] = 'true'

    labels = ['true', 'false']
    print(df['verdict'].value_counts())
    df = df.loc[df.verdict.isin(labels)]
    print("considered labels:", df.verdict.unique())
    print("after dropping invalid labels:",len(df))
    df.to_csv(data_dir+'/data.csv', sep="\t", index=False)

    #creating hash
    json_data = df.to_json().encode()
    df = df.sample(frac=1, random_state=random_state)
    df_hash = hashlib.sha256(json_data).hexdigest()

    labels_idx = [labels.index(label) for label in labels]
    labels_one_hot = np.eye(len(labels))[labels_idx]
    label_to_oh = {label:labels_one_hot[labels.index(label)] for label in labels}

    print("MEMORY: ",resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

    assert (num_folds > 2), "Needs at least three folds for Train/Dev/Test to be different from each other"
    #determine train_fold and dev_fold:
    bucket_size = int(len(data.index)/num_folds)
    fold_dev = 0 if fold_test == num_folds-1 else fold_test+1

    if not check_hash(df_hash, num_folds, drop_feat_idx=drop_feat_idx):
        lens = np.asarray([len(e.split(" ")) for e in df['body'].values])
        #IS THIS LINE NEEDED?
        #df.reset_index(drop = True, inplace = True)
        num_entries = len(df)

        #plots the data distribution by number of words
        print("Number of entries: ", num_entries)
        print("True/False: ",df.groupby('verdict').count())
        print("Mean and Std of number of words per document: ",np.mean(lens),np.std(lens), "\n")
        #input("Press any key to continue.")

        ###################################
        ############# FEATURES ############
        ###################################
        #check if new linguistic features should be generated
        flag_concat = False
        if not check_hash(df_hash, num_folds, stage="complexity") and "inf" in feature_list:
            flag_concat = True
            #Generate the features ndarray and save it to a pickle
            try:
                feat.generate_complexity()
            except Exception as e:
                print(traceback.format_exc())
                input("Error occured while GENERATING COMPLEXITY. Press any key to exit.")
                sys.exit(1)
            savehash("complexity", hashcode=df_hash)
        if not check_hash(df_hash, num_folds, stage="specificity") and "spe" in feature_list:
            flag_concat = True
            try:
                feat.generate_specificity()
            except Exception as e:
                print(traceback.format_exc())
                input("Error occured while GENERATING SPECIFICITY. Press any key to exit.")
                sys.exit(1)
            savehash("specificity", hashcode=df_hash)

        if not check_hash(df_hash, num_folds, drop_feat_idx=drop_feat_idx, stage="features") or (force_reload == 'just_reload'):
            flag_concat = True
            try:
                bool_reload = True if force_reload == "just_reload" else False
                features = feat.generateFeats(feature_list, just_reload = bool_reload)
                print(features)
                input()

            except Exception as e:
                print(traceback.format_exc())
                input("Error occured while GENERATING FEATURES. Press any key to exit.")
                sys.exit(1)
            save_p(data_dir+"/features", features)
            print("Generated Features. Saved to pickle.")
            savehash("features", hashcode=df_hash, drop_feat_idx=drop_feat_idx)

        #check if drop_features is NOT the same
        if not check_hash(df_hash, num_folds, drop_feat_idx=drop_feat_idx, stage="drop_feat"):
            flag_concat = True
            savehash("drop_feat", hashcode=df_hash, drop_feat_idx=drop_feat_idx)

        print("MEMORY AFTER FEATURES: ",resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

        ####################################
        ############### BERT ###############
        ####################################
        #check if new bert should be generated
        if not check_hash(df_hash, num_folds, stage="bert"):
            try:
                #creates the shuffle order (not random)
                index_shuf = list(range(len(df)))

                #creates a list of N=folds lists, each inner list contains the index of the elements of each fold
                bert_folds = np.array_split(index_shuf, num_folds)
                bert_folds = [a.tolist() for a in bert_folds]

                #creates an ordered list of N=entries of integers(:folds) indicating the fold idx of each entry
                fold_idx = [bert_folds.index(list(sl)) for e in index_shuf for sl in bert_folds if e in list(sl)]

                #start the bert-as-a-service server
                bert_dir = os.environ.get("BERT_BASE_DIR")
                #args = get_args_parser().parse_args(['-model_dir', bert_dir, '-port', '5555', '-port_out', '5556', '-max_seq_len', '512', '-mask_cls_sep'])
                #server = BertServer(args)
                #server.start()

                #delete the bert.csv files inside the folds
                for i in range(num_folds):
                    filename = data_dir+"/folds/"+str(i)+"/bert.csv"
                    if os.path.exists(filename):
                        subprocess.call("rm -rf "+filename, shell=True, cwd=data_dir)

                #TODO make this process read only one fold at a time
                for fold, idx in zip(fold_idx, index_shuf):
                    #generates the encodings for the texts
                    bc = BertClient(check_version=False)
                    b = bc.encode([df.body[idx]])[0]

                    bert_df = pd.DataFrame([b], columns=['f'+str(e) for e in range(len(b))])
                    bert_df.to_csv(data_dir+"/folds/"+str(fold)+"/bert.csv", mode='a+', index=False, header=False)

                #stops the bert-as-a-service server
                #shut_args = get_shutdown_parser().parse_args(['-ip','localhost','-port','5555','-timeout','5000'])
                #server.shutdown(shut_args)

            except Exception as e:
                print(traceback.format_exc())
                input("Error occured while fine training BERT. Press any key to exit.")
                sys.exit(1)

            print("BERT Embeddings Saved")
            savehash("bert", df_hash)

        #########################################
        ## CONCATENATION, SHUFFLING AND SAVING ##
        #########################################

        #if not check_hash(df_hash, num_folds, stage="concat"):
        if flag_concat:
            features = read_p(data_dir+"/features")
            features = np.delete(features,drop_feat_idx,axis=1)

            #normalize features
            features = np.nan_to_num(features)
            features_t = features.T
            for c in range(features_t.shape[0]):
                row = features_t[c]
                features_t[c] = np.interp(row, (np.min(row), np.max(row)), (-2, +2))
            features = features_t.T
            #delete labels and folds folders
            for i in range(num_folds):
                subprocess.call("rm -rf "+data_dir+"/folds/"+str(i)+"/labels", shell=True, cwd=data_dir)
                subprocess.call("rm -rf "+data_dir+"/folds/"+str(i)+"/features+bert.csv", shell=True, cwd=data_dir)
                subprocess.call("rm -rf "+data_dir+"/folds/"+str(i)+"/bert", shell=True, cwd=data_dir)
                subprocess.call("rm -rf "+data_dir+"/folds/"+str(i)+"/only_bert", shell=True, cwd=data_dir)

            #creates the shuffle order (not random)
            index_shuf = list(range(len(df)))

            #LABELS
            labels = [label_to_oh[label].tolist() for label in df['verdict'].values.tolist()]
            labels = [labels[i] for i in index_shuf]
            label_folds = np.array_split(labels, num_folds)

            for i in range(num_folds):
                fold_dir = data_dir+"/folds/"+str(i)
                if not os.path.exists(fold_dir):
                    os.mkdir(fold_dir)
                save_p(fold_dir+"/labels", label_folds[i])

            #creates a list of N=folds lists, each inner list contains the index of the elements of each fold
            bert_folds = np.array_split(index_shuf, num_folds)
            bert_folds = [a.tolist() for a in bert_folds]

            #creates an ordered list of N=entries of integers(:folds) indicating the fold idx of each entry
            fold_idx = [bert_folds.index(list(sl)) for e in index_shuf for sl in bert_folds if e in list(sl)]

            #TODO make this process read only one fold at a time
            for fold in range(num_folds):
                b_fold_csv = pd.read_csv(data_dir+"/folds/"+str(fold)+"/bert.csv", header=None)
                #gets only the indexes
                count = sum([1 for fidx,_ in zip(fold_idx, index_shuf) if fold == fidx])
                for idx in range(count):
                    b = b_fold_csv.iloc[idx]
                    entry = np.concatenate((features[idx,:],b))

                    feat_df = pd.DataFrame([entry], columns=['f'+str(e) for e in range(len(entry))])
                    feat_df.to_csv(data_dir+"/folds/"+str(fold)+"/features+bert.csv", mode='a+', index=False, header=False)

            for i in range(num_folds):
                fold_dir = data_dir+"/folds/"+str(i)
                bert = np.genfromtxt(fold_dir+"/features+bert.csv", delimiter=',')
                only_bert = np.genfromtxt(fold_dir+"/bert.csv", delimiter=',')
                print("saving bert fold ",str(i), bert.shape)
                save_p(fold_dir+"/bert", bert)
                save_p(fold_dir+"/only_bert", only_bert)

            print("MEMORY AFTER FOLDS SAVING: ",resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

            savehash("concat", hashcode=df_hash)

        checks = ["bert", "features", "concat"]
        if "inf" in feature_list: checks.append("complexity")
        if "spe" in feature_list: checks.append("specificity")

        for e in checks:
            print(e)
            print(check_hash(df_hash,num_folds,stage=e))
            if not (check_hash(df_hash,num_folds,stage=e, drop_feat_idx=drop_feat_idx)):
                print('Problem at Generation of data!')
                print("Stage: "+e)
                return

        print('Generation of data successfully done!')
        savehash("data", hashcode=df_hash)
        savehash("folds", hashcode=str(num_folds))

        return load_data(emb_type=emb_type, collapse_classes=collapse_classes, fold_test=fold_test, num_folds=num_folds, random_state=random_state, drop_feat_idx=drop_feat_idx, only_claims=only_claims)

    #check_hash is true
    else:
        print("Reading already processed data")
        #returns the selected emb type (bert/w2v)
        test_data = read_p(data_dir+"/folds/"+str(fold_test)+"/"+emb_type)
        test_target = read_p(data_dir+"/folds/"+str(fold_test)+"/labels")

        dev_data = read_p(data_dir+"/folds/"+str(fold_dev)+"/"+emb_type)
        dev_target = read_p(data_dir+"/folds/"+str(fold_dev)+"/labels")

        train_data_filenames = [data_dir+"/folds/"+str(i)+"/"+emb_type for i in range(num_folds) if i not in [fold_test,fold_dev]]
        train_data = np.concatenate([read_p(fn) for fn in train_data_filenames], axis=0)
        train_target_filenames = [data_dir+"/folds/"+str(i)+"/labels" for i in range(num_folds) if i not in [fold_test,fold_dev]]
        train_target = np.concatenate([read_p(fn) for fn in train_target_filenames], axis=0)

        return train_data, train_target, dev_data, dev_target, test_data, test_target, label_to_oh