import pycurl
import json
import time
import hmac, hashlib

from io import BytesIO

def create_time_stamp(datestr, format="%Y-%m-%d %H:%M:%S"):
    return time.mktime(time.strptime(datestr, format))

class poloniex:
    API_URL = 'https://54.95.147.104'
    API_VERSION = 'v1'

    def __init__(self, APIKey, Secret):
        self.APIKey = APIKey
        self.Secret = Secret
        self.session = self._init_session()

    def _init_session(self):
        session = pycurl.Curl()
        session.setopt(pycurl.SSL_VERIFYPEER, 0)
        session.setopt(pycurl.SSL_VERIFYHOST, 0)
        session.setopt(pycurl.TCP_NODELAY, 1)
        session.setopt(pycurl.TCP_KEEPALIVE, 1)
        return session

    def _order_params_for_sig(self, data):
        strs = []
        for key in sorted(data.keys()):
            strs.append("%s=%s" % (key, data[key]))
        return '&'.join(strs)

    def post_process(self, before):
        after = before

        # Add timestamps if there isnt one but is a datetime
        if 'return' in after and isinstance(after['return'], list):
                for x in xrange(0, len(after['return'])):
                    if isinstance(after['return'][x], dict):
                        if 'datetime' in after['return'][x] and 'timestamp' not in after['return'][x]:
                            after['return'][x]['timestamp'] = float(create_time_stamp(after['return'][x]['datetime']))

        return after

    def api_query(self, method, command, req={}):

        session = self.session
        # Prepare the basic headers
        headers = ['User-Agent: ']

        must_process = False
        req['command'] = command
        if method == "get":
            url = "https://poloniex.com/public"
            data_string = self._order_params_for_sig(req)
            if data_string != "":
                url = "%s?%s" % (url, data_string)
            session.setopt(pycurl.HTTPGET, True)

        else:
            must_process = True
            req['nonce'] = int(time.time() * 1000)
            data_string = self._order_params_for_sig(req)
            sign = hmac.new(self.Secret, data_string, hashlib.sha512).hexdigest()
            url = "https://poloniex.com/tradingApi"
            session.setopt(pycurl.POSTFIELDS, data_string)
            session.setopt(pycurl.POST, True)
            headers.append('Sign: %s' % sign)
            headers.append('Key: %s' % self.APIKey)

        # Apply the settings
        buff = BytesIO()
        session.setopt(pycurl.HTTPHEADER, headers)
        session.setopt(pycurl.URL, url)
        session.setopt(pycurl.WRITEDATA, buff)
        session.perform()

        # Handle the response
        json_ret = json.loads(buff.getvalue())
        if must_process:
            return self.post_process(json_ret)
        return json_ret

    def return_ticker(self):
        return self.api_query(
            "get",
            "returnTicker"
        )

    def return_24volume(self):
        return self.api_query(
            "get",
            "return24Volume"
        )

    def return_order_book(self, currency_pair, depth=1):
        return self.api_query(
            "get",
            "returnOrderBook",
            {
                'currency_pair': currency_pair,
                'depth': '%d' % depth
            }
        )

    def return_market_trade_history(self, currency_pair):
        return self.api_query(
            "get",
            "returnTradeHistory",
            {
                'currency_pair': currency_pair
            }
        )

    def return_chart_data(self, currency_pair, start, end):
        return self.api_query(
            "get",
            "returnChartData",
            {
                'currency_pair': currency_pair,
                'start': start,
                'end': end
            }
        )

    def return_currencies(self):
        return self.api_query("get", "returnCurrencies")

    def return_loan_orders(self):
        return self.api_query("get", "returnLoadOrders")

    # Returns all of your balances.
    # Outputs:
    # {"BTC":"0.59098578","LTC":"3.31117268", ... }
    def return_balances(self):
        return self.api_query(
            'post',
            'returnBalances'
        )

    # Returns your open orders for a given market, specified by the "currency_pair" POST parameter, e.g. "BTC_XCP"
    # Inputs:
    # currency_pair  The currency pair e.g. "BTC_XCP"
    # Outputs:
    # order_number   The order number
    # type          sell or buy
    # rate          Price the order is selling or buying at
    # Amount        Quantity of order
    # total         Total value of order (price * quantity)
    def return_open_orders(self, currency_pair):
        return self.api_query(
            'post',
            'returnOpenOrders',
            {
                "currency_pair": currency_pair
            }
        )

    # Returns your trade history for a given market, specified by the "currency_pair" POST parameter
    # Inputs:
    # currency_pair  The currency pair e.g. "BTC_XCP"
    # Outputs:
    # date          Date in the form: "2014-02-19 03:44:59"
    # rate          Price the order is selling or buying at
    # amount        Quantity of order
    # total         Total value of order (price * quantity)
    # type          sell or buy
    def return_trade_history(self, currency_pair):
        return self.api_query(
            'post',
            'returnTradeHistory',
            {
                "currency_pair": currency_pair
            }
        )

    # Places a buy order in a given market. Required POST parameters are "currency_pair", "rate", and "amount". If successful, the method will return the order number.
    # Inputs:
    # currency_pair  The currency pair
    # rate          price the order is buying at
    # amount        Amount of coins to buy
    # Outputs:
    # order_number   The order number
    def buy(self, currency_pair, rate, amount):
        return self.api_query(
            'post',
            'buy',
            {
                "currency_pair": currency_pair,
                "rate": rate,
                "amount": amount
            }
        )

    # Places a sell order in a given market. Required POST parameters are "currency_pair", "rate", and "amount". If successful, the method will return the order number.
    # Inputs:
    # currency_pair  The currency pair
    # rate          price the order is selling at
    # amount        Amount of coins to sell
    # Outputs:
    # order_number   The order number
    def sell(self, currency_pair, rate, amount):
        return self.api_query(
            'post',
            'sell',
            {
                "currency_pair": currency_pair,
                "rate": rate,
                "amount": amount
            }
        )

    # Cancels an order you have placed in a given market. Required POST parameters are "currency_pair" and "order_number".
    # Inputs:
    # currency_pair  The currency pair
    # order_number   The order number to cancel
    # Outputs:
    # succes        1 or 0
    def cancel(self, currency_pair, order_number):
        return self.api_query(
            'post',
            'cancelOrder',
            {
                "currency_pair": currency_pair,
                "order_number": order_number
            }
        )

    # Immediately places a withdrawal for a given currency, with no email confirmation. In order to use this method, the withdrawal privilege must be enabled for your API key. Required POST parameters are "currency", "amount", and "address". Sample output: {"response":"Withdrew 2398 NXT."}
    # Inputs:
    # currency      The currency to withdraw
    # amount        The amount of this coin to withdraw
    # address       The withdrawal address
    # Outputs:
    # response      Text containing message about the withdrawal
    def withdraw(self, currency, amount, address):
        return self.api_query(
            'post',
            'withdraw',
            {
                "currency": currency,
                "amount": amount,
                "address": address
            }
        )


import numpy
import cPickle
import argparse
from config import *
import random
import re
import json
import logging
import os

logging.basicConfig(level = logging.DEBUG,
	                format = '[%(asctime)s %(levelname)s] %(message)s',
	                datefmt = '%d %b %H:%M:%S')

parser = argparse.ArgumentParser("generate vocabulary file and pickled corpus")
parser.add_argument('-c', '--config', help = 'path to config file')
parser.add_argument('-wd', '--withdict', action = "store_true", help = 'preprocess with existing vocabulary file')
parser.add_argument('-gr', '--fromgr', action = "store_true", help = 'transfer from Groundhog vocabulary file, must use with --wd')
parser.add_argument('-np', '--nopreprocess', action = "store_true", help = 'skip preprocessing')
parser.add_argument('-ns', '--noshuffle', action = "store_true", help = 'skip shuffling')

def preprocess(num, \
		       vocab_f, \
			   ivocab_f, \
			   input_f, \
			   output_f, \
			   data_vocab = 'cPickle', \
			   data_corpus = 'json', \
			   withdict = False, \
			   fromgr = False):
	'''
		Count the most frequent words in the training corpus.
		Then, generate the vocabulary file and index the corpus.

		:type vocab_f: string
		:param vocab_f: the path to vocabulary file

		:type ivocab_f: string
		:param ivocab_f: the path to vocabulary to index file

		:type input_f: string
		:param input_f: the path to corpus (text) file

		:type output_f: string
		:param output_f: the path to indexed corpus file

		:type withdict: bool
		:param withdict: if set to True, vocabulary will be loaded from existing file instead.
	'''
	if not withdict:
		logging.info('Building dictionary')
		input = open(input_f, 'r')
		line = input.readline()
		count = {}

		# building a dictionary
		while line != '':
			line = line.strip()
			words = re.split(' ', line)
			for word in words:
				if not count.has_key(word):
					count[word] = 1
				else:
					count[word] += 1

			line = input.readline()

		count_sort = sorted(count.items(), key=lambda x: x[1], reverse = True)
		vocab = ['<s>', 'UNK']
		ivocab = {'UNK': 1, '<s>': 0}
		vnum = 2
		while vnum < num:
			if vnum-2 >= len(count_sort):
				break
			vocab.append(count_sort[vnum - 2][0])
			ivocab[count_sort[vnum - 2][0]] = vnum
			vnum += 1

		with open(vocab_f, 'wb') as f:
			cPickle.dump(vocab, f)
		with open(ivocab_f, 'wb') as f:
			cPickle.dump(ivocab, f)

		input.close()
	else:
		logging.info('Loading dictionary')
		vocab_old = cPickle.load(open(vocab_f, 'rb'))
		ivocab_old = cPickle.load(open(ivocab_f, 'rb'))

		if fromgr:
			vocab = [0]*num
			for key in ivocab_old:
				vocab[key] = ivocab_old[key]
			vocab[0] = '<s>'

			ivocab = vocab_old
			if ivocab.has_key('</s>'):
				ivocab['<s>'] = ivocab['</s>']
				del ivocab['</s>']

			with open(vocab_f, 'wb') as f:
				cPickle.dump(vocab, f)
			with open(ivocab_f, 'wb') as f:
				cPickle.dump(ivocab, f)
		else:
			vocab = vocab_old
			ivocab = ivocab_old

	# indexing the corpus
	logging.info('Indexing the corpus')
	input = open(input_f, 'r')
	line = input.readline()
	corpora = []
	num_words = 0
	UNKs = 0
	num_sentences = 0
	while line != '':
		corpora.append([])
		line = line.strip()
		words = line.split(' ')
		for word in words:
			if not ivocab.has_key(word):
				corpora[num_sentences].append(1)
				UNKs += 1
			else:
				corpora[num_sentences].append(ivocab[word])
			num_words += 1
		num_sentences += 1
		line = input.readline()

	if not withdict:
		logging.info('%d unique words in %d sentences' % (len(count_sort), num_sentences))
		logging.info('Perserving %d words (including the UNK and empty symbols) and covering %2f of the words.' % (len(vocab), 100. * (num_words - UNKs) / num_words))
	if data_corpus == 'cPickle':
		with open(output_f, 'wb') as f:
			cPickle.dump(corpora, f)
	elif data_corpus == 'json':
		with open(output_f, 'wb') as f:
			json.dump(corpora, f)

def shuffle(src, trg, src_shuf, trg_shuf, data_corpus = 'json'):
	'''
		Randomly shuffling the parallel corpus

		:type src: string
		:param src: the path to indexed source corpus file

		:type trg: string
		:param trg: the path to indexed target corpus file

		:type src_shuf: string
		:param src_shuf: the path to shuffled indexed source corpus file

		:type trg_shuf: string
		:param trg_shuf: the path to shuffled indexed target corpus file
	'''

	# source
	logging.info('Loading source corpus')
	if data_corpus == 'cPickle':
		before_source = cPickle.load(open(src, 'rb'))
	elif data_corpus == 'json':
		before_source = json.load(open(src, 'rb'))
	length = len(before_source)
	logging.info('Total number of source sentences: %d' % length)

	idx = range(length)
	random.shuffle(idx)
	logging.info('Shuffling source corpus')
	after_source = []
	for i in range(len(idx)):
		if (i + 1) % 100000 == 0:
			logging.debug(str(i + 1))
		index = idx[i]
		after_source.append(before_source[index])

	logging.info('Dumping shuffled source corpus')

	if data_corpus == 'cPickle':
		with open(src_shuf, 'wb') as f:
			cPickle.dump(after_source, f)
			f.close()
	elif data_corpus == 'json':
		with open(src_shuf, 'wb') as f:
			json.dump(after_source, f)
			f.close()

	del before_source
	del after_source

	# target
	logging.info('Loading target corpus')
	if data_corpus == 'cPickle':
		before_target = cPickle.load(open(trg, 'rb'))
	elif data_corpus == 'json':
		before_target = json.load(open(trg, 'rb'))
	assert length == len(before_target)

	logging.info('Shuffling target corpus')
	after_target = []
	for i in range(len(idx)):
		if (i+1) % 100000 == 0:
			logging.debug(str(i+1))
		index = idx[i]
		after_target.append(before_target[index])

	logging.info('Dumping shuffled target corpus')

	if data_corpus == 'cPickle':
		with open(trg_shuf, 'wb') as f:
			cPickle.dump(after_target, f)
	elif data_corpus == 'json':
		with open(trg_shuf, 'wb') as f:
			json.dump(after_target, f)

def shuffle_mono(src, src_shuf, data_corpus = 'json'):
	'''
		Randomly shuffling the monolingual corpus.

		:type src: string
		:param src: the path to indexed corpus file

		:type src_shuf: string
		:param src_shuf: the path to indexed corpus file
	'''

	if data_corpus == 'cPickle':
		before_source = cPickle.load(open(src, 'rb'))
	elif data_corpus == 'json':
		before_source = json.load(open(src, 'rb'))
	length = len(before_source)

	idx = range(length)
	random.shuffle(idx)
	after_source = []
	for i in range(len(idx)):
		if (i + 1) % 100000 == 0:
			logging.debug(str(i + 1))
		index = idx[i]
		after_source.append(before_source[index])

	if data_corpus == 'cPickle':
		with open(src_shuf, 'wb') as f:
			cPickle.dump(after_source, f)
			f.close()
	elif data_corpus == 'json':
		with open(src_shuf, 'wb') as f:
			json.dump(after_source, f)
			f.close()

if __name__ == "__main__":
	args = parser.parse_args()

	#init config
	cfg = config()
	if args.config:
		cfg = update_config(cfg, load_config(open(args.config, 'r').read()))

	logging.info('STEP 1: Processing\n')
	if os.path.exists(cfg['src_shuf']) and os.path.exists(cfg['trg_shuf']):
		logging.info('File exists! Skipped.\n')
	else:
		if not args.nopreprocess:
			logging.info('STEP 1.1: Processing source corpus')
			preprocess(cfg['index_eos_src'],
					   cfg['vocab_src'],
					   cfg['ivocab_src'],
					   cfg['src_text'],
					   cfg['src'],
					   cfg['data_vocab'],
					   cfg['data_corpus'],
					   args.withdict,
					   args.fromgr)
			logging.info('Done!\n')
			logging.info('STEP 1.2: Processing target corpus')
			preprocess(cfg['index_eos_trg'],
				   	   cfg['vocab_trg'],
					   cfg['ivocab_trg'],
					   cfg['trg_text'],
					   cfg['trg'],
					   cfg['data_vocab'],
					   cfg['data_corpus'],
					   args.withdict,
					   args.fromgr)
			logging.info('Done!\n')

		if not args.noshuffle:
			logging.info('STEP 1.3: Shuffling')
			shuffle(cfg['src'],
				    cfg['trg'],
				    cfg['src_shuf'],
				    cfg['trg_shuf'],
				    cfg['data_corpus'])
			logging.info('Done!\n')

	if cfg['src_mono_shuf'] != '':
		if os.path.exists(cfg['src_mono_shuf']):
			logging.info('Monolingual source file exists! Skipped.\n')
		else:
			logging.info('STEP 1.4: Processing monolingual source corpus')
			preprocess(cfg['index_eos_src'],
					   cfg['vocab_src'],
					   cfg['ivocab_src'],
					   cfg['src_mono_text'],
					   cfg['src_mono'],
					   cfg['data_vocab'],
					   cfg['data_corpus'],
					   True,
					   args.fromgr)
			logging.info('Shuffling\n')
			shuffle_mono(cfg['src_mono'],
					     cfg['src_mono_shuf'],
					     cfg['data_corpus'])
			logging.info('Done!\n')
	if cfg['trg_mono_shuf'] != '':
		if os.path.exists(cfg['trg_mono_shuf']):
			logging.info('Monolingual target file exists! Skipped.\n')
		else:
			logging.info('STEP 1.5: Processing monolingual target corpus.')
			preprocess(cfg['index_eos_trg'],
				 	   cfg['vocab_trg'],
					   cfg['ivocab_trg'],
					   cfg['trg_mono_text'],
					   cfg['trg_mono'],
					   cfg['data_vocab'],
					   cfg['data_corpus'],
					   True,
					   args.fromgr)
			logging.info('Shuffling\n')
			shuffle_mono(cfg['trg_mono'],
					     cfg['trg_mono_shuf'],
					     cfg['data_corpus'])
			logging.info('Done!\n')


from collections import OrderedDict
import yaml

def get_configs(dataset):
    data = yaml.load(open('dataset_cfg.yaml'))
    return data[dataset]

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)
    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=0):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / (.0001 + self.count)

    def __str__(self):
        """String representation for logging
        """
        # for values that should be recorded exactly e.g. iteration number
        if self.count == 0:
            return str(self.val)
        # for stats
        return '%.4f (%.4f)' % (self.val, self.avg)

class LogCollector(object):
    """A collection of logging objects that can change from train to val"""

    def __init__(self):
        # to keep the order of logged variables deterministic
        self.meters = OrderedDict()

    def update(self, k, v, n=0):
        # create a new meter if previously not recorded
        if k not in self.meters:
            self.meters[k] = AverageMeter()
        self.meters[k].update(v, n)

    def __str__(self):
        """Concatenate the meters in one log line
        """
        s = ''
        for i, (k, v) in enumerate(iter(self.meters.items())):
            if i > 0:
                s += '  '
            s += k + ' ' + str(v)
        return s

    def tb_log(self, tb_logger, prefix='', step=None):
        """Log using tensorboard
        """
        for k,v in iter(self.meters.items()):
            tb_logger.log_value(prefix + k, v.val, step=step)


import numpy as np
from scipy.interpolate import interp1d
from matplotlib import pyplot as plt

# Cosine 함수를 0부터 10pi까지 20개 만든다.
x = np.linspace(0, 10 * np.pi, 20)
y = np.cos(x)

# interoperate 함수로 보간법을 적용하여 linear(선형보정) quadratic(부드러운 보정) 두가지 방법으로 만든다
fl = interp1d(x, y, kind='linear')
fq = interp1d(x, y, kind='quadratic')

xint = np.linspace(x.min(), x.max(), 1000)
yintl = fl(xint)
yintq = fq(xint)

# Plot the data and the interpolation
plt.plot(x, y, 'bo')  # input value 청색점으로 표현
plt.plot(xint, yintl, color='green', linewidth=2) # linear 값 녹색 직선으로 표현
plt.plot(xint, yintq, color='red', linewidth=2) # quadratic 값 적색 곡선으로 표현
plt.legend(['Input', 'Linear', 'Quadratic'])
plt.ylim(-2, 2)

plt.title('Interoperate')
plt.show()


from typing import Optional, List

def find_nth(haystack: str, needle: str, n: int, start:Optional[int]=None, end:Optional[int]=None) -> int:
    offset, end = start if start else 0, end if end else len(haystack)

    i = haystack.find(needle, offset, end)
    while i != -1 and n > 1:
        i = haystack.find(needle, i + len(needle), end)
        n -= 1
    return i

def decompress(s: str) -> str:
    """
    Description:
        In this exercise, you're going to decompress a compressed string.
        Your input is a compressed string of the format number[string] and the decompressed output form should be the
        string written number times. For example:

        3[abc]4[ab]c → abcabcabcababababc

        - Number can have more than one digit. For example, 10[a] is allowed, and just means aaaaaaaaaa
        - One repetition can occur inside another. For example, 2[3[a]b] decompresses into aaabaaab
        - Characters allowed as input include digits, small English letters and brackets [ ].
        - Digits are only to represent amount of repetitions.
        - Letters are just letters.
        - Brackets are only part of syntax of writing repeated substring.
        - Input is always valid, so no need to check its validity.

    Link:
        https://techdevguide.withgoogle.com/paths/advanced/compress-decompression/

    Args:
        s: compressed string.

    Returns:
        Decompressed string.
    """
    def _decompress(start: int, end: int) -> List[str]:
        left = s.find('[', start, end)
        first_right = s.find(']', start, end)

        if left == first_right == -1: return list(s[start:end])  # pierscin: only letters in []
        if s[left:first_right] == '': return []  # pierscin: empty string in []

        count_of_left = s.count('[', start, first_right)
        right = find_nth(s, ']', count_of_left, start, end)

        res = []
        for i in range(start, left):
            if s[i].isalpha(): res.append(s[i])
            else:
                number = int(s[i:left])
                break
        else:
            number = 1

        if '[' in s[left + 1: right + 1]:  # pierscin: nested expression
            res += number * _decompress(left + 1, right + 1)
        else:
            res += number * _decompress(left + 1, right) + _decompress(right + 1, end)

        # pierscin: letters after brackets
        i = right + 1
        while i < end and s[i].isalpha():
            res.append(s[i])
            i += 1

        return res

    s = '1[' + s[:] + ']'

    return ''.join(_decompress(0, len(s)))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Mar 27 14:45:03 2022

@author: luisa
"""
#------------------------------------------------------------------------------
#1. IMPORTAR LIBRERIAS
import pandas as pd
import  numpy as np
from datetime import datetime
from tqdm import tqdm # libreria para saber el tiempo de ejecución
from sqlalchemy import create_engine
import os
import math
#-----------------------------------------------------------------------------
#2. LECTURAS DE DATOS
# 0-CodigoEstacion
# 1-CodigoSensor
# 2-FechaObservacion
# 3-ValorObservado
# 4-NombreEstacion
# 5-Departamento
# 6-Municipio
# 7-ZonaHidrografica
# 8-Latitud
# 9-Longitud
# 10-DescripcionSensor
# 11-UnidadMedida

#CodigoEstacion,FechaObservacion,ValorObservado,NombreEstacion,Departamento,Municipio,ZonaHidrografica,Latitud,Longitud

#-----------------------------------------------------------------------------
#3. FUNCIONES

#3.1 EXTRACCIÓN DE DATOS DE DATABASE
# Load de data
def SQL_PD(table_or_sql,eng):
       engine = create_engine(eng)
       conn = engine.connect()
       generator_object = pd.read_sql(table_or_sql,con=conn)
       return generator_object

#-----------------------------------------------------------------------------

#4. ANALIZAR EL FUNCIONAMIENTO DEL SQL_PD

my_query='''
SELECT CodigoEstacion,FechaObservacion,ValorObservado,NombreEstacion,Departamento,
Municipio,ZonaHidrografica,Latitud,Longitud
FROM precipitacion
WHERE (codigoestacion = '25025380')
'''
eng = 'sqlite:///DATA.db'

#Se carga el DF con las especificaciones anteriores
df_25025380 = SQL_PD(my_query,eng)
#Convertir a datatime
df_25025380["fecha"]=pd.to_datetime(df_25025380['FechaObservacion']).dt.strftime("%d/%m/%Y %X")
#organizar las filas de mayor a menor con respecto a la fecha
df_25025380 = df_25025380.sort_values(by='fecha')
#Se resetean los indices
df_25025380=df_25025380.reset_index(drop=True)
#longitud de filas
n=len(df_25025380)
#longitud de columnas
shape = df_25025380.shape
#Obtener el nombre de las columnas
columns_names = df_25025380.columns.values

print("Estación 25025380")
print("1. La fecha inicial =",df_25025380["fecha"][0] )
print("2. La fecha final =",df_25025380["fecha"][n-1] )
print("3. La cantidad de filas y columnas =",shape )
print("4. El nombre de las columnas es=",columns_names)

df_25025380.head()
df_25025380.tail()
df_25025380.index()

print("1. La fecha inicial para la estación 25025380 es",df_25025380["fecha"][0] )

df_25025380

m1=sorted(df_25025380["fecha"], reverse=False)
m1=pd.DataFrame(m1)

m1.tail()
#df_25025380["fecha"].sorted()

# Leer varias estaciones
#DataFrame con codigos de estacion

my_query1='''
SELECT CodigoEstacion
FROM precipitacion
WHERE (codigoestacion = '25025380')
'''

codigo = SQL_PD(my_query1,eng)

for i in tqdm(range(3)):

    cod = codigo["CodigoEstacion"][i]

    my_query2='''
    SELECT CodigoEstacion,FechaObservacion,ValorObservado,NombreEstacion,Departamento,
    Municipio,ZonaHidrografica,Latitud,Longitud
    FROM precipitacion
    WHERE (codigoestacion = {})
    '''.format(int(cod))

    df_est = SQL_PD(my_query2,eng)

    df_est["fecha"]=pd.to_datetime(df_est['FechaObservacion']).dt.strftime("%d/%m/%Y %X")
    #organizar las filas de mayor a menor con respecto a la fecha
    df_est = df_est.sort_values(by='fecha')
    #Se resetean los indices
    df_est=df_est.reset_index(drop=True)
    #longitud de filas
    n=len(df_est)
    #longitud de columnas
    shape = df_est.shape
    #Obtener el nombre de las columnas
    columns_names = df_est.columns.values

    print("Estación",cod)
    print("1. La fecha inicial =",df_est["fecha"][0] )
    print("2. La fecha final =",df_est["fecha"][n-1] )
    print("3. La cantidad de filas y columnas =",shape )
    print("4. El nombre de las columnas es=",columns_names)

    del df_est


# Tim Momose, October 2014

# ----------------------------------------------------------------------
#  Imports
# ----------------------------------------------------------------------
import scipy
import numpy as np
from copy import deepcopy
# SUAVE Imports
from SUAVE.Core import Data, Data_Exception, Data_Warning
#from SUAVE.Methods.Aerodynamics.Fidelity_Zero.Drag.parasite_drag_aircraft import parasite_drag_aircraft
# SUAVE-AVL Imports
from .Data.Inputs   import Inputs
from .Data.Wing     import Wing, Section, Control_Surface
from .Data.Body     import Body
from .Data.Aircraft import Aircraft
from .Data.Cases    import Run_Case
from .Data.Configuration import Configuration

def create_avl_datastructure(geometry,conditions):

	avl_aircraft      = translate_avl_geometry(geometry)
	avl_configuration = translate_avl_configuration(geometry,conditions)
	avl_cases         = translate_avl_cases(conditions,settings.run_cases)
	#avl_cases         = setup_test_cases(conditions)

	# pack results in a new AVL inputs structure
	avl_inputs = Inputs()
	avl_inputs.aircraft      = avl_aircraft
	avl_inputs.configuration = avl_configuration
	avl_inputs.cases         = avl_cases
	return avl_inputs

def translate_avl_geometry(geometry):

	aircraft = Aircraft()
	aircraft.tag = geometry.tag

	# FOR NOW, ASSUMING THAT CONTROL SURFACES ARE NOT ALIGNED WITH WING SECTIONS (IN THIS CASE, ROOT AND TIP SECTIONS)
	for wing in geometry.wings:
		w = translate_avl_wing(wing)
		aircraft.append_wing(w)

	for body in geometry.fuselages:
		b = translate_avl_body(body)
		aircraft.append_body(b)

	return aircraft

def translate_avl_wing(suave_wing):

	w = Wing()
	w.tag       = suave_wing.tag
	w.symmetric = suave_wing.symmetric
	w.vertical  = suave_wing.vertical
	w.sweep     = suave_wing.sweep
	w.dihedral  = suave_wing.dihedral
	w = populate_wing_sections(w,suave_wing)

	return w

def translate_avl_body(suave_body):

	b = Body()
	b.tag       = suave_body.tag
	b.symmetric = True #body.symmetric
	b.lengths.total = suave_body.lengths.total
	b.lengths.nose  = suave_body.lengths.nose
	b.lengths.tail  = suave_body.lengths.tail
	b.widths.maximum = suave_body.width
	b.heights.maximum = suave_body.heights.maximum
	b = populate_body_sections(b,suave_body)

	return b

def populate_wing_sections(avl_wing,suave_wing):
	symm     = avl_wing.symmetric
	sweep    = avl_wing.sweep
	dihedral = avl_wing.dihedral
	span     = suave_wing.spans.projected
	semispan = suave_wing.spans.projected * 0.5 * (2 - symm)
	origin   = suave_wing.origin
	root_section = Section()
	root_section.tag    = 'root_section'
	root_section.origin = origin
	root_section.chord  = suave_wing.chords.root
	root_section.twist  = suave_wing.twists.root

	tip_section = Section()
	tip_section.tag  = 'tip_section'
	tip_section.chord = suave_wing.chords.tip
	tip_section.twist = suave_wing.twists.tip
	tip_section.origin = [origin[0]+semispan*np.tan(sweep),origin[1]+semispan,origin[2]+semispan*np.tan(dihedral)]

	if avl_wing.vertical:
		temp = tip_section.origin[2]
		tip_section.origin[2] = tip_section.origin[1]
		tip_section.origin[1] = temp

	avl_wing.append_section(root_section)
	avl_wing.append_section(tip_section)

	if suave_wing.control_surfaces:
		for ctrl in suave_wing.control_surfaces:
			num = 1
			for section in ctrl.sections:
				semispan_fraction = (span/semispan) * section.origins.span_fraction
				s = Section()
				s.chord  = scipy.interp(semispan_fraction,[0.,1.],[root_section.chord,tip_section.chord])
				s.tag    = '{0}_section{1}'.format(ctrl.tag,num)
				s.origin = section.origins.dimensional
				s.origin[0] = s.origin[0] - s.chord*section.origins.chord_fraction
				s.twist  = scipy.interp(semispan_fraction,[0.,1.],[root_section.twist,tip_section.twist])
				c = Control_Surface()
				c.tag     = ctrl.tag
				c.x_hinge = 1. - section.chord_fraction
				c.sign_duplicate = ctrl.deflection_symmetry

				s.append_control_surface(c)
				avl_wing.append_section(s)
				num += 1

	return avl_wing

def populate_body_sections(avl_body,suave_body):

	symm = avl_body.symmetric
	semispan_h = avl_body.widths.maximum * 0.5 * (2 - symm)
	semispan_v = avl_body.heights.maximum * 0.5
	origin = suave_body.origin
	root_section = Section()
	root_section.tag    = 'center_horizontal_section'
	root_section.origin = origin
	root_section.chord  = avl_body.lengths.total

	tip_section = Section()
	tip_section.tag  = 'outer_horizontal_section'
	nl = avl_body.lengths.nose
	tl = avl_body.lengths.tail
	tip_section.origin = [origin[0]+nl,origin[1]+semispan_h,origin[2]]
	tip_section.chord = root_section.chord - nl - tl

	avl_body.append_section(root_section,'horizontal')
	avl_body.append_section(tip_section,'horizontal')
	tip_sectionv1 = deepcopy(tip_section)
	tip_sectionv1.origin[1] = origin[1]
	tip_sectionv1.origin[2] = origin[2] - semispan_v
	tip_sectionv1.tag       = 'lower_vertical_section'
	avl_body.append_section(tip_sectionv1,'vertical')
	avl_body.append_section(root_section,'vertical')
	tip_sectionv2 = deepcopy(tip_section)
	tip_sectionv2.origin[1] = origin[1]
	tip_sectionv2.origin[2] = origin[2] + semispan_v
	tip_sectionv2.tag       = 'upper_vertical_section'
	avl_body.append_section(tip_sectionv2,'vertical')

	return avl_body

def translate_avl_configuration(geometry,conditions):

	config = Configuration()
	config.reference_values.sref = geometry.reference_area
	config.reference_values.bref = geometry.wings['Main Wing'].spans.projected
	config.reference_values.cref = geometry.wings['Main Wing'].chords.mean_aerodynamic
	config.reference_values.cg_coords = geometry.mass_properties.center_of_gravity

	#config.parasite_drag = 0.0177#parasite_drag_aircraft(conditions,configuration,geometry)

	config.mass_properties.mass = geometry.mass_properties.max_takeoff ###
	moment_tensor = geometry.mass_properties.moments_of_inertia.tensor
	config.mass_properties.inertial.Ixx = moment_tensor[0][0]
	config.mass_properties.inertial.Iyy = moment_tensor[1][1]
	config.mass_properties.inertial.Izz = moment_tensor[2][2]
	config.mass_properties.inertial.Ixy = moment_tensor[0][1]
	config.mass_properties.inertial.Iyz = moment_tensor[1][2]
	config.mass_properties.inertial.Izx = moment_tensor[2][0]

	#No Iysym, Izsym assumed for now

	return config

def translate_avl_cases(conditions,suave_cases):

	runcases = Run_Case.Container()

	for case in suave_cases:
		kase = Run_Case()
		kase.tag = case.tag
		kase.conditions.mach  = case.conditions.freestream.mach
		kase.conditions.v_inf = case.conditions.freestream.velocity
		kase.conditions.rho   = case.conditions.freestream.density
		kase.conditions.g     = case.conditions.freestream.gravitational_acceleration
		kase.angles.alpha     = case.conditions.aerodynamics.angle_of_attack
		kase.angles.beta      = case.conditions.aerodynamics.side_slip_angle
		kase.parasite_drag    = case.conditions.aerodynamics.parasite_drag

		for deflect in case.conditions.stability_and_control.control_surface_deflections:
			kase.append_control_deflection(deflect.tag,deflect.magnitude)

		runcases.append_case(case)

	return runcases

def setup_test_cases(conditions):

	runcases = Run_Case.Container()

	alphas = [-10,-5,-2,0,2,5,10,20]
	mach   = conditions.freestream.mach
	v_inf  = conditions.freestream.velocity
	rho    = conditions.density
	g      = conditions.g
	for alpha in alphas:
		case = Run_Case()
		case.tag = 'Alpha={}'.format(alpha)
		case.conditions.mach  = mach
		case.conditions.v_inf = v_inf
		case.conditions.rho   = rho
		case.conditions.gravitation_acc = g
		case.angles.alpha     = alpha
		runcases.append_case(case)

	return runcases


import numpy as np
import csv
import abc

from keras.utils.np_utils import to_categorical

class CSVLoader(object):

    __metaclass__ = abc.ABCMeta

    def __init__(self, data_dir, config):

        self.config = config
        self.images_label_pairs = []
        self.input_shape = tuple(config["input_shape"])

        with open(data_dir, "rb") as csvfile:
            for (file_path, label)in list(csv.reader(csvfile)):
                self.images_label_pairs.append((file_path, int(label)))

    def get_data(self, should_shuffle=True, is_prediction=False):

        start = 0

        while True:

            data_batch = np.zeros((self.config["batch_size"], ) + self.input_shape)  # (batch_size, cols, rows, channels)
            label_batch = np.zeros((self.config["batch_size"], self.config["num_classes"]))  # (batch_size,  num_classes)

            for i, (file_path, label) in enumerate(self.images_label_pairs[start:start + self.config["batch_size"]]):

                data = self.process_file(file_path)
                height, width, channels = data.shape
                data_batch[i, : height, :width, :] = data
                label_batch[i, :] = to_categorical([label], nb_classes=self.config["num_classes"]) # one-hot encoding

            start += self.config["batch_size"]

            # Reset generator
            if start + self.config["batch_size"] > self.get_num_files():
                start = 0
                if should_shuffle:
                    np.random.shuffle(self.images_label_pairs)

            # For predicitions only return the data
            if is_prediction:
                yield data_batch
            else:
                yield data_batch, label_batch

    def get_input_shape(self):

        return self.input_shape

    def get_num_files(self):

        # Minimum number of data points without overlapping batches
        return (len(self.images_label_pairs) // self.config["batch_size"]) * self.config["batch_size"]

    def get_labels(self):

        return [label for (file_path, label) in self.images_label_pairs]

    @abc.abstractmethod
    def process_file(self, file_path):

        raise NotImplementedError("Implement in child class.")


#!/usr/bin/env python
import numpy as np
from scipy import linalg
from scipy.sparse import diags
from numpy.linalg import inv
def main():

	#A
	n = 10
	k = np.array([-1*np.ones(n-2),-1*np.ones(n-1),10*np.ones(n),-1*np.ones(n-1),-1*np.ones(n-2)])
	offset = [-2,-1,0,1,2]
	A = diags(k,offset).toarray()
	#print(A)

	#D
	n = 10
	k = np.array([8*np.ones(n)])
	offset = [0]
	D = diags(k,offset).toarray()
	#print(D)

	#M
	n = 10
	k = np.array([-1*np.ones(n-2),-1*np.ones(n-1),1*np.ones(n),0*np.ones(n-1),0*np.ones(n-2)])
	offset = [-2,-1,0,1,2]
	M = diags(k,offset).toarray()
	#print(M)

	#N
	n = 10
	k = np.array([0*np.ones(n-2),0*np.ones(n-1),1*np.ones(n),-1*np.ones(n-1),-1*np.ones(n-2)])
	offset = [-2,-1,0,1,2]
	N = diags(k,offset).toarray()
	#print(N)
	#print(M+N+D)
	#print(A)
#(M+D)x(k+1)=-Nx(k)+b
	m=M+D
	#print(m)
	m1=inv(np.matrix(m))
	#print(m1)
	n=np.dot(-1,N)
	main_m=np.dot(m1,n)
	##=print("main_m is:",main_m)
	eigenvalues = np.linalg.eigvals(main_m)
	#print("eigenvalues are:",eigenvalues)
	#iterations
	s=(10,1)
	b=np.ones(s)
	c=np.dot(m1,b)
	x=linalg.solve(A, b)
	print("the aswer is:",x)
	iteration=0
	x1 = np.ones((10,1))
	err = np.ones((10,1))*100
	#print(np.dot(m1,b))
	while np.max(err) > 1:
		iteration=iteration+1
		xn=np.dot(main_m,x1)+c
		err = abs((xn - x1)/(xn+0.00001))*100
		x1 = xn

	print("first:",x1)
	print(iteration)

#Dx(k+1)= -(M+N)x(k)+b
	m1=M+N
	m2=np.dot(-1,m1)
	d=inv(np.matrix(D))
	main_m2=np.dot(d,m2)
	print("main_m2:",main_m2)
	eigenvalues2 = np.linalg.eigvals(main_m2)
	print("eigenvalues are:",eigenvalues2)
	#iteration
	iteration2=0
	x2 = np.ones((10,1))
	err2 = np.ones((10,1))*100
	while np.max(err2) > 0:
		iteration2=iteration2+1
		xn2 = np.dot(main_m2,x2)+np.dot(d,b)
		err2 = abs((xn2 - x2)/xn2)*100
		x2 = xn2
	print("second:",x2)
	print(iteration2)

#(M+N)x(k+1)=-Dx(k)+b

	t=M+N

	m3=inv(np.matrix(t))

	r=np.dot(-1,D)
	main_m3=np.dot(m3,r)

	eigenvalues3 = np.linalg.eigvals(main_m3)
	print("eigenvalues are:",eigenvalues3)
	print("not converging")

if __name__ == '__main__':
	main()


from sqlalchemy import select, insert, delete, update
from sqlalchemy.orm import sessionmaker
from .DB_models import *
from .DB_engines import engine

session_factory = sessionmaker(bind=engine)

class Populate:
    @classmethod
    def get_list_of_items(cls):
        """Returns DISTINCT 'names' from table 'Items'"""
        session = session_factory()
        list_of_items = session.execute(
            select(Items.name)
            .distinct()
        )
        for row in list_of_items:
            return row

    @classmethod
    def populate_active_items(cls, name):
        """Returns oldest items of requested name whose amount is not 0"""
        session = session_factory()
        item = session.scalars(
            select(
                Items.ID,
                Items.name,
                Items.amount,
                Items.pricePerUnit)
                .join(Invoices)
                .filter(Items.name.in_([name]), Items.amount != 0)
                .order_by(Invoices.date)
            ).first()
        return item

    @classmethod
    def populate_ingredients(cls):
        session = session_factory()
        ingredients = session.scalars(
            select(Ingredients)
            .order_by(Ingredients.name)
        ).all()
        return ingredients

    @classmethod
    def populate_recipes(cls):
        session = session_factory()
        recipes = session.scalars(
            select(Recipe)
            .order_by(Recipe.name)
        ).all()
        return recipes

    @classmethod
    def populate_promotions(cls):
        session = session_factory()
        promotions = session.scalars(
            select(Promotion)
            .order_by(Promotion.name)
        ).all()
        return promotions

    @classmethod
    def populate_groups(cls):
        session = session_factory()
        groups = session.scalars(
            select(Group)
            .order_by(Group.name)
        ).all()
        return groups

class POST:
    @classmethod
    def create_ingredient(cls, data):
        session = session_factory()
        name, item_name, used_amount, \
        production_cost, sale_price, menu = data

        new_ingredient = Ingredients(
            name            = name,
            item_name       = item_name,
            used_amount     = used_amount,
            production_cost = production_cost,
            sale_price      = sale_price,
            menu            = menu
        )
        session.add_all([new_ingredient])
        try:
            session.commit()
            return (200, "Ingredient created successfully")
        except:
            session.rollback()
            return (401, "Failed to create ingredient")
        finally:
            session.close()

    @classmethod
    def create_recipe(cls, data):
        session = session_factory()
        name, ingredient_list, production_cost, sale_price, \
        recipe_amount, portions, menu = data

        new_recipe = Ingredients(
            name            = name,
            ingredient_list = ingredient_list,
            production_cost = production_cost,
            sale_price      = sale_price,
            recipe_amount   = recipe_amount,
            portions        = portions,
            menu            = menu
        )
        session.add_all([new_recipe])
        try:
            session.commit()
            return (200, "Recipe created successfully")
        except:
            session.rollback()
            return (401, "Failed to create recipe")
        finally:
            session.close()

    @classmethod
    def create_promotion(cls, data):
        session = session_factory()
        name, production_cost, sale_price, items_list, menu = data

        new_promotion = Ingredients(
            name            = name,
            production_cost = production_cost,
            sale_price      = sale_price,
            items_list      = items_list,
            menu            = menu
        )
        session.add_all([new_promotion])
        try:
            session.commit()
            return (200, "Promotion created successfully")
        except:
            session.rollback()
            return (401, "Failed to create promotion")
        finally:
            session.close()

    @classmethod
    def create_group(cls, data):
        session = session_factory()
        name, members, menu = data

        new_group = Ingredients(
            name            = name,
            members         = members,
            menu            = menu
        )
        session.add_all([new_group])
        try:
            session.commit()
            return (200, "Promotion created successfully")
        except:
            session.rollback()
            return (401, "Failed to create promotion")
        finally:
            session.close()

class PUT:
    @classmethod
    def update_ingredient(cls, data):
        session = session_factory()
        name, item_name, used_amount, production_cost, sale_price, menu = data

        update = session.execute(
                    select(Ingredients)
                    .filter(Ingredients.name == name)
                ).scalar_one()
        update.name = name
        update.item_name = item_name
        update.used_amount = used_amount
        update.production_cost = production_cost
        update.sale_price = sale_price
        update.menu = menu

        try:
            session.commit()
            return (200, "Ingredient updated successfully")
        except:
            session.rollback()
            return (401, "Failed to update ingredient")
        finally:
            session.close()

    @classmethod
    def update_recipe(cls, data):
        session = session_factory()
        name, ingredients_list, production_cost, sale_price, recipe_amount, portion, menu = data

        update = session.execute(
                    select(Recipe)
                    .filter(Recipe.name == name)
                ).scalar_one()
        update.name            = name
        ingredients_list       = ingredients_list
        update.production_cost = production_cost
        update.sale_price      = sale_price
        update.recipe_amount   = recipe_amount
        update.portions        = portion
        update.menu            = menu

        try:
            session.commit()
            return (200, "Recipe updated successfully")
        except:
            session.rollback()
            return (401, "Failed to update recipe")
        finally:
            session.close()

    @classmethod
    def update_promotion(cls, data):
        session = session_factory()
        name, production_cost, sale_price, items_list, menu = data

        update = session.execute(
            select(Promotion)
            .filter(Promotion.name == name)
        ).scalar_one()
        update.name = name
        update.production_cost = production_cost
        update.sale_price = sale_price
        update.items_list = items_list
        update.menu = menu

        try:
            session.commit()
            return (200, "Recipe updated successfully")
        except:
            session.rollback()
            return (401, "Failed to update recipe")
        finally:
            session.close()

    @classmethod
    def update_group(cls, data):
        session = session_factory()
        name , members, menu = data

        update = session.execute(
            select(Group)
            .filter(Group.name == name)
        ).scalar_one()
        update.name = name
        update.members = members
        update.menu = menu

        try:
            session.commit()
            return (200, "Recipe updated successfully")
        except:
            session.rollback()
            return (401, "Failed to update recipe")
        finally:
            session.close()

class DELETE:
    @classmethod
    def delete_ingredient(cls, name):
        session = session_factory()
        session.execute(
            delete(Ingredients)
            .where(Ingredients.name == name)
        )
        try:
            session.commit()
            return (200, "Ingredient deleted successfully")
        except:
            session.rollback()
            return (401, "Failed to delete ingredient")

    @classmethod
    def delete_recipe(cls, name):
        session = session_factory()
        session.execute(
            delete(Recipe)
            .where(Recipe.name == name)
        )
        try:
            session.commit()
            return (200, "Recipe deleted successfully")
        except:
            session.rollback()
            return (401, "Failed to delete recipe")

    @classmethod
    def delete_promotion(cls, name):
        session = session_factory()
        session.execute(
            delete(Promotion)
            .where(Promotion.name == name)
        )
        try:
            session.commit()
            return (200, "Promotion deleted successfully")
        except:
            session.rollback()
            return (401, "Failed to delete promotion")

    @classmethod
    def delete_group(cls, name):
        session = session_factory()
        session.execute(
            delete(Group)
            .where(Group.name == name)
        )
        try:
            session.commit()
            return (200, "Group deleted successfully")
        except:
            session.rollback()
            return (401, "Failed to delete group")

Populate.get_list_of_items()


# -*- coding: utf-8 -*-
"""LDR Importer GPLv2 license.

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

"""

from datetime import datetime

class Console:

    @staticmethod
    def __makeMessage(msg, prefix=None):
        """Construct the message for displaying in the console.

        Formats, timestamps, and identifies the message
        as coming from this script.

        @param {Tuple} msg The message to be displayed.
        @param {String} prefix Any text to prefix to the message.
        @return {String} The constucted message.
        """
        msg = [str(text) for text in msg]

        # Prefix text if needed
        if prefix:
            msg.insert(0, str(prefix))

        return "\n[LDR Importer] ({0}) {1}".format(
            datetime.now().strftime("%H:%M:%S.%f")[:-4], " ".join(msg))

    @staticmethod
    def log(*msg):
        """Print logging messages to the console.

        @param {Tuple} msg The message to be displayed.
        """
        print(Console.__makeMessage(msg))

    @staticmethod
    def warn(*msg):
        """Print warning messages to the console.

        @param {Tuple} msg The message to be displayed.
        """
        print(Console.__makeMessage(msg, "Warning!"))


# sqlite.py

from ..chain_links import Module

import sqlite3

class SQLiteModule(Module):
    def __init__(self):
        super().__init__()
        self.connection = None

    def setup_db(self, filename):
        """
        Sets up a new database for this module with the name `filename`.

        Note that you must also call `connect_to_db` after calling
        this since it closes the connection to make sure everything
        is saved correctly.
        """
        self.connection = sqlite3.connect(filename)
        c = self.connection.cursor()
        c.execute("CREATE TABLE sentiments (input text, sentiment text)")
        self.connection.commit()
        self.connection.close()

    def connect_to_db(self, filename):
        self.connection = sqlite3.connect(filename)

    def process(self, data):
        text = data['text']
        s = data['classification']

        c = self.connection.cursor()
        c.execute("INSERT INTO sentiments VALUES (?, ?)", (text, s))

        self.connection.commit()


"""
	This file containts several data analysis methods designed to run on a Dataset Class
	(PCA, clustering, some classification tests)

	TODO: add these to the example-scripts folder to make some preliminary analysis easier
"""

import logging
import typing

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from matplotlib import cm

log = logging.getLogger(__name__)
#pylint: disable=import-outside-toplevel #Make scikit-learn optional

def plot_subset_columns(dataset, columns, label_column_name = "Classification"):
	"""
	Quick plot of 3d plot of 3 columns of a dataframe
	"""
	fig = plt.figure(figsize = (10,10))
	ax = fig.add_subplot(1, 1, 1, projection='3d') #3d plot #pylint: disable=invalid-name
	ax.set_xlabel(columns[0], fontsize = 15) #set labels
	ax.set_ylabel(columns[1], fontsize = 15)
	ax.set_zlabel(columns[2], fontsize = 15)
	ax.set_title('Columns:' + columns[0] + " "+ columns[1] + " " + columns[2] , fontsize = 20) #set title
	classes = dataset.df[label_column_name].unique() #get all classes
	colors = cm.rainbow(np.linspace(0,1,len(classes))) #get new color for each one #type: ignore #pylint: disable=no-member
	legend = []

	for target, color in zip(classes,colors):
		kept_indices = dataset.df['Classification'] == target #Split on classification type
		ax.scatter(dataset.df.loc[kept_indices, columns[0]], #plot the columns
				dataset.df.loc[kept_indices, columns[1]],
				dataset.df.loc[kept_indices, columns[2]],
				c = color.reshape(1,-1), #Color each type according to different color wheel
				s = 50)
		legend.append(dataset.classes[int(target)])
	ax.legend(legend)
	ax.grid()
	plt.show()

def get_pca(df, components =3, scale=True ):  #pylint: disable=invalid-name
	"""
	Uses PCA to add a "PC1", "PC2", "PC3" column to the passed dataframe and returns it
	Args:
		df (pd.DataFrame): The dataframe to perform PCA on
		components (int, optional): Number of components to return. Defaults to 3.
		scale (bool, optional): Wether to scale the data before PCA. Defaults to True.
	"""
	from sklearn import (decomposition, preprocessing)
	log.info("Generating 3d PCA plot...")
	df_pca = df.copy()
	if scale: # SCikit learn automatically centers, but does not scale data (MinMaxScaler is used here)
		for col in df_pca: #go over columns, normalize them TODO: what about time? class?
			scaler = preprocessing.MinMaxScaler()
			norm_col = scaler.fit_transform(df_pca[[col]]) #Min-max normalize data (it is centered by scikit-pca)
			df_pca.loc[:, col] = norm_col

	pca = decomposition.PCA(n_components=components)

	df_pca = df_pca.dropna()  #drop NaN for PCA analysis
	princ_components = pca.fit_transform(df_pca) #Apply PCA
	principal_df = pd.DataFrame(data = princ_components, columns = ['PC1', 'PC2', 'PC3'], index=df_pca.index)

	return principal_df

def plot_pca_2d(dataset):
	"""
	Create a 2D PCA plot of the dataset
	"""
	pca_result_df  = get_pca(
		dataset.df[dataset.df.columns.difference(["Date", "Time", "Classification", "DateTime"])], components=2)
	pca_result_df = pd.concat([pca_result_df, dataset.df['Classification']]) #append classification column to PCA plot
	classes = range(int(pca_result_df['Classification'].max()) + 1) #        \
	fig = plt.figure(figsize = (10,10))
	ax = fig.add_subplot(1, 1, 1) #pylint: disable=invalid-name
	ax.set_xlabel('Principal Component 1', fontsize = 15)
	ax.set_ylabel('Principal Component 2', fontsize = 15)
	ax.set_title('2 component PCA', fontsize = 20)
	legend = []

	colors = cm.rainbow(np.linspace(0,1,len(classes))) #get color for each one #type: ignore #pylint: disable=no-member
	for target, color in zip(classes,colors):
		kept_indices = pca_result_df['Classification'] == target #Split on classification type
		ax.scatter(pca_result_df.loc[kept_indices, 'PC1'],
				pca_result_df.loc[kept_indices, 'PC2'],
				c = color.reshape(1,-1), #Color each type according to different color wheel
				s = 50)
		legend.append(dataset.classes(target))

	ax.legend(legend)
	ax.grid()
	plt.show()

def plot_pca_3d(
		dataframe : pd.DataFrame,
		plot_only_classes : typing.Optional[typing.List[str]] = None,
		dt_column = "DateTime",
		lbl_column = None,
		figurenr : int = 5,
		figsize : tuple = (15, 10)):
	"""Plots a 3D PCA splot for the provided dataframe (all columns included).
	Colors each dot according to its column

	Args:
		dataframe (pd.DataFrame): The dataframe with all data
		plot_only_classes (Typing.List[str] optional): Wether to only plot certain columns (classes), provide a
			list of string. Defaults to [].
	"""
	if plot_only_classes is None:
		plot_only_classes = []

	#Perform PCA on all passed data (except datetime/class label):
	pca_result_df = get_pca(dataframe[dataframe.columns.difference([dt_column, lbl_column])], components=3)

	classes = ["All"]
	if lbl_column:
		pca_result_df = pd.concat([pca_result_df, dataframe[lbl_column]]) #append label column to PCA plot
		classes = dataframe[lbl_column].unique()

	fig = plt.figure(figurenr)
	ax = fig.add_subplot(1, 1, 1, projection='3d') #pylint: disable=invalid-name

	fig.set_size_inches(figsize[0], figsize[1])

	ax.set_xlabel('Principal Component 1', fontsize = 15)
	ax.set_ylabel('Principal Component 2', fontsize = 15)
	ax.set_zlabel('Principal Component 3', fontsize = 15)
	ax.set_title('3 Component PCA', fontsize = 20)

	log.debug(f"Classes: {classes}")

	colors = cm.rainbow(np.linspace(0,1,len(classes))) #get color for each one #type: ignore #pylint: disable=no-member
	legend = []
	# colors = ['r', 'g', 'b']
	for target, color in zip(classes,colors):
		log.debug(f"Now processing class {target}")
		# if target == 6: #Skip jetting as this process applies to
		#     continue
		if len(plot_only_classes) > 0 and target not in plot_only_classes: #If plot
			continue

		if lbl_column is None: #If all should be added:
			kept_indices = dataframe.index
		else:
			kept_indices = dataframe[lbl_column] == target #Split on classification type
			legend.append(target)

		if len(pca_result_df.loc[kept_indices]) <= 0:
			# print("None")
			log.warning(f"No entries for class {target}")
			continue

		ax.scatter(pca_result_df.loc[kept_indices, 'PC1'],
				pca_result_df.loc[kept_indices, 'PC2'],
				pca_result_df.loc[kept_indices, 'PC3'],
				c = color.reshape(1,-1), #Color each type according to different color wheel
				s = 50)
	ax.legend(legend)
	ax.grid()
	fig.show()

def plot_df_scatter_class(
		df, #pylint: disable=invalid-name
		ax,	#pylint: disable=invalid-name
		color_column="Classification",
		legend_label_prefix = "",
		class_name_dict = None,
		plot_kwargs = None
	):
	"""
	Plot a dataframe with a classification column, color each class differently
	Args:
		df (pd.DataFrame): The dataframe to plot
		ax (matplotlib.axes.Axes): The ax to plot on
		color_column (str, optional): The column to use for coloring. Defaults to "Classification".
		legend_label_prefix (str, optional): Prefix to add to each class label in the legend. Defaults to "".
		class_name_dict (dict, optional): Dictionary to translate class labels to other labels. Defaults to {-1 : None}.
	"""
	if plot_kwargs is None:
		plot_kwargs = {}
	if class_name_dict is None:
		class_name_dict = {-1 : None}
	dims = len(df.columns) - 1 #Columns count - classification column (determines color)
	data_cols = df.columns[ df.columns != color_column]

	if len(data_cols) > 4:
		log.fatal(f"Trying to generate a plot with dims={dims} (excluding color_column), this is impossible, exiting...")
		raise ValueError(f"Trying to generate a plot with dims={dims} (excluding color_column), "
			"this is impossible, exiting...")

	# classes = range(int(df[color_column].max()) + 1)
	classes = df[color_column].unique() #get all unique classes

	colormap = cm.rainbow(np.linspace(0,1,len(classes))) #get colors #type: ignore #pylint: disable=no-member

	class_color_dict = {}

	for i, _ in enumerate(classes):
		class_color_dict[i] = colormap[i] #create color mapping of form {"class": [r,g,b,a], etc}
	colors = pd.Series(df.loc[:, color_column]).map(class_color_dict) #create color array for each entry

	if dims == 2:
		df.plot.scatter(x=data_cols[0], y=data_cols[1], c=colors, ax=ax, **plot_kwargs)
	elif dims == 3:
		for cur_class in classes:
			cur_class_df = df[df[color_column] == cur_class] #Select only
			cur_label = legend_label_prefix + str(cur_class)
			if cur_label in class_name_dict.keys(): #if label-translation
				cur_label = class_name_dict[cur_label] #type: ignore
			ax.scatter(
				cur_class_df[data_cols[0]].to_numpy(),
	      		cur_class_df[data_cols[1]].to_numpy(),
				cur_class_df[data_cols[2]].to_numpy(),
				**plot_kwargs,
				label=cur_label
			)
			ax.legend()

		ax.set_xlabel(data_cols[0], fontsize = 10) #set labels
		ax.set_ylabel(data_cols[1], fontsize = 10)
		ax.set_zlabel(data_cols[2], fontsize = 10)
	else:
		print("Dims = 1, cant plot")

	ax.grid()
	# plt.show()

def k_cluster(df, n_clusters = 4): #pylint: disable=invalid-name
	"""Cluster using k-means"""
	import sklearn.cluster
	df = df[df.columns.difference(["Date", "Time", "DateTime", "Classification"])]
	df = df.dropna() #remove nan values
	model = sklearn.cluster.KMeans(n_clusters=n_clusters).fit(df) #Train using the trainset (split[0]), without labels
	results = pd.Series(model.predict(df), name="Cluster", index=df.index) #Nearest clusters
	df['Cluster'] = results
	return df

def k_means_classification(dataset, label_column_name= "Classification", n_clusters=100, n_splits = 5):
	"""
	Do k-means clustering, then determine the label of that cluster using majority vote. We can then use this
	to classify new data points by finding the nearest cluster and using the majority vote label of that cluster.
	"""
	import sklearn
	import sklearn.cluster
	from sklearn import (model_selection)
	from sklearn.metrics import classification_report
	#Simple k-means classification using majority-vote for each cluster center, contains a test as well
	df_norm = dataset.get_df_norm()
	train_df = df_norm[df_norm.columns.difference(["Date", "Time"])] #Train on all (normalized) data except
		# date, time and label
	train_df = train_df.dropna() #remove nan values
	print("Number of detected classification labels in database: "
    	+  str(len(dataset.df[label_column_name].unique())) + "  (" + str(dataset.df[label_column_name].unique()) + ")")
	splits = model_selection.KFold(n_splits = n_splits, shuffle=False) #TODO: shuffle manually to equal-shared?
	labels_actual = np.array([])
	labels_predicted = np.array([])
	splits = splits.split(train_df) #get splits for training dataframe

	# for n_test in range(n_splits): #perform classification and evaluate for every split
	for n_test, split_indexes in enumerate(splits):
		print(" Doing K-fold test "+ str(n_test+1) + " out of "+  str(n_splits))
		# split_indexes = next(splits) #get split indexes
		split = train_df.iloc[split_indexes[0]], train_df.iloc[split_indexes[1]] #get train and testsplit

		train_split = split[0][split[0].columns.difference([label_column_name])] #get current trainsplit
		test_split = split[1][split[1].columns.difference([label_column_name])] #get current testsplit

		model = sklearn.cluster.KMeans(n_clusters=n_clusters).fit(train_split) #Train using the trainset (split[0]),
		results = pd.Series(model.predict(train_split), name="Cluster") #create a result series with the nearest cluster
		#create df that has class label+cluster for each entry, index should first be reset or join will fail:
		df_label_and_cluster = pd.concat([split[0][label_column_name].reset_index(drop=True), results], axis=1)
		cluster_label = ["UNKNOWN" for i in range(len(results.unique()))]
		#create dataframe with each center and the (majority voted) label for that center:
		df_center_labels = pd.DataFrame(columns=["Center", "Label Mode"])

		for cur_cluster in results.unique(): #Go over each mean (cluster), do majority vote to get labels per cluster
			cluster_df = df_label_and_cluster[df_label_and_cluster["Cluster"] == cur_cluster] #get all
			cluster_label = cluster_df[label_column_name].mode() #do majority vote for cluster
			cluster_row = pd.DataFrame([[cur_cluster, cluster_label.iloc[0]]], columns=["Center", "Label Mode"])
			df_center_labels = df_center_labels.append(cluster_row) #append row #type: ignore

		#----------------------Testing-----------------------
		#Get current predictions/labels in numpy array
		cur_pred = pd.DataFrame(model.predict(test_split), columns=["Center"])
		cur_pred = cur_pred.merge(df_center_labels, on=["Center"], how='left')["Label Mode"] #join to get majority-vote
		cur_pred_np = np.array(cur_pred, dtype=int)

		cur_act = split[1][label_column_name].reset_index(drop=True)
		cur_act_np = np.array(cur_act, dtype=int)

		#append to total list of actual/predictions
		labels_actual = np.append(labels_actual, cur_act_np) #create list with all actual labels
		labels_predicted = np.append(labels_predicted, cur_pred_np) #create list with all predicted labels

	labels_actual = [ dataset.classes[int(labels_actual[i])] for i in range(len(labels_actual))] #translate to int+class
	labels_predicted = [ dataset.classes[int(labels_predicted[i])] for i in range(len(labels_predicted))]

	print(classification_report(labels_actual, labels_predicted))


import json
import tensorflow as tf
from tensorflow import keras
from imutils import paths
from PIL import Image as im
import matplotlib.pyplot as plt

import pandas as pd
import numpy as np
import imageio
import cv2
import os
import glob
import time
import random
import pickle
train_df = pd.DataFrame()     #train 파일 내의 모든 json 경로와 라벨값
test_df = pd.DataFrame()      #test 파일 내의 모든 json 경로와 라벨값
val_df = pd.DataFrame()       #val 파일 내의 모든 json 경로와 라벨값

MAX_IMAGE_WIDTH = 320
MAX_IMAGE_HEIGHT = 180
MAX_IMAGE_SIZE = MAX_IMAGE_WIDTH * MAX_IMAGE_HEIGHT
RESCALE=6

MAX_IMAGE_WIDTH = 320
MAX_IMAGE_HEIGHT = 180
RESCALE = 6
BATCH_SIZE = 64
EPOCHS = 10

MAX_SEQ_LENGTH = 20
NUM_FEATURES = 2048
json_path ='/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/ALL_DATA/json/'  #기본 json 폴더 위치
base_path='/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/AI_HUB_1.03/input/main/'   #text 파일 위치

def get_file_list(check):        #파일 리스트와 라벨 리스트를 동시에 얻는 함수
    file_list = []
    label_index = []
    if check == 'valfile.txt':
        path = base_path + check
    elif check =='testfile.txt':
        path = base_path + check
    elif check =='trainfile.txt':
        path = base_path + check

    ff = open(path ,'r')
    lines = ff.readlines()
    num = len(lines)
    for line in lines:
        label = line.split(' ')[1]
        line = line.split(' ')[0]
        file_list.append(json_path+line+'.json')
        label_index.append(label)
    return file_list , label_index

a ,b = get_file_list('trainfile.txt')
train_df['file_name'] =a
train_df['label'] =b
c,d = get_file_list('testfile.txt')
test_df['file_name'] =c
test_df['label'] =d
e,f = get_file_list('valfile.txt')
val_df['file_name'] =e
val_df['label'] =f

test_end = len(test_df)
train_end=len(train_df)

def make_bitmap(a,b):
    #A = random.randint(0,10000)
    targetList = [0] * MAX_IMAGE_SIZE
    for i in range(len(a)):
        setBoldPoint(targetList, a[i], b[i])
    #targetArray = np.array(targetList,dtype = np.uint8)
    targetArray = np.fromiter(targetList, dtype=np.uint8)
    targetArray= np.reshape(targetArray, (MAX_IMAGE_HEIGHT,MAX_IMAGE_WIDTH))
    data = im.fromarray(targetArray)
    data = data.convert('RGB')
    #a = np.array(data)
    #print(a.shape)
    #data.save('./img/'+str(A)+'.png')
    # saving the final output as a PNG file
    data.save('gfg_dummy_pic.png')

    return data

def setBoldPoint(targetList, x,y):
    if x<0 or y <0 or x >= MAX_IMAGE_WIDTH or y >= MAX_IMAGE_HEIGHT:
        return
    if y-1 >= 0 and x-1 >= 0 : #x와 y가 1보다 크면
        targetList[MAX_IMAGE_WIDTH * (y-1) + (x-1)] = 255
    if y-1 >= 0 :  #y좌표가 1보다 크면
        targetList[MAX_IMAGE_WIDTH * (y-1) + (x)] = 255
    if y-1 >= 0 and x+1 < MAX_IMAGE_WIDTH:
        targetList[MAX_IMAGE_WIDTH * (y-1) + (x+1)] = 255
    if x-1 >= 0 :
        targetList[MAX_IMAGE_WIDTH * (y) + (x-1)] = 255
    targetList[MAX_IMAGE_WIDTH * (y) + (x)] = 255
    if x+1 < MAX_IMAGE_WIDTH:
        targetList[MAX_IMAGE_WIDTH * (y) + (x+1)] = 255
    if y+1 < MAX_IMAGE_HEIGHT and x-1 >= 0 :
        targetList[MAX_IMAGE_WIDTH * (y+1) + (x-1)] = 255
    if y+1 < MAX_IMAGE_HEIGHT:
        targetList[MAX_IMAGE_WIDTH * (y+1) + (x)] = 255
    if y+1 < MAX_IMAGE_HEIGHT and  x+1 < MAX_IMAGE_WIDTH:
        targetList[MAX_IMAGE_WIDTH * (y+1) + (x+1)] = 255
    return targetList

def extract_skeleton(start,end,dataframe, choice):  #json에서 skeleton point를 뽑는다.
    s_time = time.time()
    train_file = '/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/ALL_DATA/json/Action/test/A01/C011_A01_SY02_P07_S03_02DBS.json'
    #train_json = dataframe['file_name'].to_list()
    train_json = []
    train_json.append(train_file)
    empty_list_x=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
    empty_list_y=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
    image_frame_list= []
    total_file_object_list=[]
    start = 0
    end =1
    for i in range(start, end): #전체에서
        print(train_json[i]+' json 파일에서 이미지 전처리중')
        with open(train_json[i],'r') as f: #동영상 하나
            json_file = json.load(f)
        image_frame_list=[]
        for frame in json_file['file'][0]['frames']:  #동영상 하나
            point1 = []
            point2 = []
            point3 = []
            point4 = []

            person_length = len(frame['persons'])
            if person_length ==0:
                data = make_bitmap(empty_list_x+empty_list_x,empty_list_y+empty_list_y)
            elif person_length==1:
                a=frame['persons'][0]['keypoints']
                point1 = [int(float(point.split(',')[0])//RESCALE) for point in a]
                point2 = [int(float(point.split(',')[1])//RESCALE) for point in a]
                data = make_bitmap(point1+empty_list_x , point2+empty_list_y)
            else:
                a=frame['persons'][0]['keypoints']
                b=frame['persons'][1]['keypoints']

                point1 = [int(float(point.split(',')[0])//RESCALE) for point in a]
                point2 = [int(float(point.split(',')[1])//RESCALE) for point in a]
                point3 = [int(float(point.split(',')[0])//RESCALE) for point in b]
                point4 = [int(float(point.split(',')[1])//RESCALE) for point in b]

                data = make_bitmap(point1+point3,point2+point4)
            image_frame_list.append(data)
        print(train_json[i])
        name = train_json[i].split('/')[-1]
        name=name.split('.')[0]
        #with open('./pickle_'+choice+'/'+name+'.pkl', 'wb') as f :
             #pickle.dump(image_frame_list , f)
        #total_file_object_list.append(image_frame_list)
        #print(total_file_object_list)
        e_time = time.time()
        print(e_time - s_time)
    #return total_file_object_list

#total_file_object_list = extract_skeleton(0,train_end,train_df,'train')
#extract_skeleton(0,train_end,train_df,'train')
#total_file_test_object_list = extract_skeleton(0,test_end,test_df, 'test')
extract_skeleton(0,test_end,train_df, 'test')
#print(len(total_file_object_list))
#rint(len(total_file_object_list[0]))
#print(train_df['file_name'].tolist())

"""

def build_feature_extractor():
    feature_extractor = keras.applications.ResNet50(
        weights="imagenet",
        include_top=False,
        pooling="avg",
        input_shape=(MAX_IMAGE_HEIGHT, MAX_IMAGE_WIDTH, 3),
    )
    preprocess_input = keras.applications.resnet.preprocess_input

    inputs = keras.Input((MAX_IMAGE_HEIGHT, MAX_IMAGE_WIDTH, 3))
    preprocessed = preprocess_input(inputs)

    outputs = feature_extractor(preprocessed)
    return keras.Model(inputs, outputs, name="feature_extractor")

feature_extractor = build_feature_extractor()

label_processor = tf.keras.layers.experimental.preprocessing.StringLookup(
    num_oov_indices=0, vocabulary=np.unique(train_df["label"])
)

def prepare_all_videos(df, check):
    s_time = time.time()
    num_samples = len(df)
    json_paths = df["file_name"].values.tolist()
    #json_paths = json_paths[0:5]
    labels = df["label"].values
    labels = label_processor(labels[..., None]).numpy()

    # `frame_masks` and `frame_features` are what we will feed to our sequence model.
    # `frame_masks` will contain a bunch of booleans denoting if a timestep is
    # masked with padding or not.
    frame_masks = np.zeros(shape=(num_samples, MAX_SEQ_LENGTH), dtype="bool")
    frame_features = np.zeros(
        shape=(num_samples, MAX_SEQ_LENGTH, NUM_FEATURES), dtype="float32"
    )
    #['a', 'b', 'c']
    # For each video.
    if check =='train':
        pkl_list = glob.glob('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pickle_train/*.pkl')
    elif check =='test':
        pkl_list = glob.glob('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pickle_test/*.pkl')

    for idx, path in enumerate(pkl_list): #7193개의 아이템이 있는 리스트    path 한  동영상내의 이미지 모음(3600개)
        e_time = time.time()
        print(e_time-s_time)
        # Gather all
        # its frames and add a batch dimension.
        #print(path)
        pkl_file_name = path.split('/')[-1]
        with open(path,'rb') as f:
            data = pickle.load(f)

        bitmap_object_np = np.array([np.array(file) for file in data])
        print(bitmap_object_np.shape)
        frames = bitmap_object_np[None, ...]

        # Initialize placeholders to store the masks and features of the current video.
        temp_frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype="bool")
        temp_frame_features = np.zeros(
            shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype="float32"
        )

        # Extract features from the frames of the current video.
        for i, batch in enumerate(frames):
            video_length = batch.shape[0]
            length = min(MAX_SEQ_LENGTH, video_length)
            for j in range(length):
                temp_frame_features[i, j, :] = feature_extractor.predict(
                    batch[None, j, :]
                )
            temp_frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked

        frame_features[idx,] = temp_frame_features.squeeze()
        frame_masks[idx,] = temp_frame_mask.squeeze()
    #print(idx)
    return frame_features, frame_masks, labels

train_frame_features,train_frame_masks, train_labels = prepare_all_videos(train_df, 'train')
with open('./pkl_train/'+'train_frame_features.pkl', 'wb') as f :
    pickle.dump(train_frame_features ,f)
with open('./pkl_train/'+'train_frame_masks.pkl', 'wb') as f :
    pickle.dump(train_frame_masks ,f)
with open('./pkl_train/'+'train_labels.pkl', 'wb') as f :
    pickle.dump(train_labels ,f)

test_frame_features,test_frame_masks, test_labels = prepare_all_videos(test_df, 'test')
with open('./pkl_test/'+'test_frame_features.pkl', 'wb') as f :
    pickle.dump(test_frame_features ,f)
with open('./pkl_test/'+'test_frame_masks.pkl', 'wb') as f :
    pickle.dump(test_frame_masks ,f)
with open('./pkl_test/'+'test_labels.pkl', 'wb') as f :
    pickle.dump(test_labels ,f)

with open('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pkl_train/train_frame_features.pkl','rb') as f:
    tr_frame_features = pickle.load(f)
with open('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pkl_train/train_frame_masks.pkl','rb') as f:
    tr_frame_masks = pickle.load(f)
with open('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pkl_train/train_labels.pkl','rb') as f:
    tr_frame_label = pickle.load(f)
train_data = (tr_frame_features,tr_frame_masks)
train_labels = tr_frame_label

with open('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pkl_test/test_frame_features.pkl','rb') as f:
    te_frame_features = pickle.load(f)
print(type(te_frame_features))
with open('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pkl_test/test_frame_masks.pkl','rb') as f:
    te_frame_masks = pickle.load(f)
print(type(te_frame_masks))
with open('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pkl_test/test_labels.pkl','rb') as f:
    te_label = pickle.load(f)
print(type(te_label))
test_data=(te_frame_features,te_frame_masks)
test_labels =te_label

# Utility for our sequence model.
def get_sequence_model():

    class_vocab = label_processor.get_vocabulary()

    frame_features_input = keras.Input((MAX_SEQ_LENGTH, NUM_FEATURES))
    mask_input = keras.Input((MAX_SEQ_LENGTH,), dtype="bool")

    # Refer to the following tutorial to understand the significance of using `mask`:
    # https://keras.io/api/layers/recurrent_layers/gru/
    x = keras.layers.GRU(16, return_sequences=True)(
        frame_features_input, mask=mask_input
    )
    x = keras.layers.GRU(8)(x)
    x = keras.layers.Dropout(0.4)(x)
    x = keras.layers.Dense(8, activation="relu")(x)
    output = keras.layers.Dense(len(class_vocab), activation="softmax")(x)

    rnn_model = keras.Model([frame_features_input, mask_input], output)

    rnn_model.compile(
        loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"]
    )
    return rnn_model

# Utility for running experiments.
def run_experiment():
    filepath = "/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/tmp/video_classifier"
    file_name = '/1_08cp_10.ckpt'
    checkpoint = keras.callbacks.ModelCheckpoint(filepath+file_name, save_weights_only=True, monitor = 'loss' , save_best_only=True, verbose=1)

    seq_model = get_sequence_model()
    history = seq_model.fit(
        [train_data[0], train_data[1]],
        train_labels,
        #validation_split=0.3,
        epochs=EPOCHS,
        callbacks=[checkpoint],
    )

    seq_model.load_weights(filepath+file_name)
    _, accuracy = seq_model.evaluate([test_data[0], test_data[1]], test_labels)
    print(f"Test accuracy: {round(accuracy * 100, 2)}%")

    return history, seq_model
_, sequence_model = run_experiment()

def prepare_single_video(frames):
    #print(frames)
    frames = frames[None, ...]
    frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype="bool")
    frame_features = np.zeros(shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype="float32")

    for i, batch in enumerate(frames):
        video_length = batch.shape[0]
        length = min(MAX_SEQ_LENGTH, video_length)
        for j in range(length):
            frame_features[i, j, :] = feature_extractor.predict(batch[None, j, :])
        frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked

    return frame_features, frame_mask

txt =[]
def sequence_prediction(path):

    class_vocab = label_processor.get_vocabulary()
    #print(class_vocab)
    frames = path
    #print(frames)
    frame_features, frame_mask = prepare_single_video(frames)
    probabilities = sequence_model.predict([frame_features, frame_mask])[0]

    for i in np.argsort(probabilities)[::-1]:
        print(f"  {class_vocab[i]}: {probabilities[i] * 100:5.2f}%")
        print(f"  {class_vocab[1]}: {probabilities[1] * 100:5.2f}%")
        print(f"  {class_vocab[2]}: {probabilities[2] * 100:5.2f}%")
        print(f"  {class_vocab[3]}: {probabilities[3] * 100:5.2f}%")
        print(f"  {class_vocab[4]}: {probabilities[4] * 100:5.2f}%")
        txt.append(class_vocab[i]+'\n')
        break

    return frames

#print(test_videos)
test_list= glob.glob('/workspace/NIA/NIA_AI_DATASET_2021-ST-GCAE/E2ON/source/CNN_RNN_model/pickle_test/*.pkl')
for i in range(1):
    print(i)
    try:
        with open(test_list[i],'rb') as f:
            test_videos = pickle.load(f)

    except:
        pass
    else:
        test_video = np.array([np.array(a) for a in test_videos])
        test_frames = sequence_prediction(test_video)
with open('result.txt', 'w') as file:    # hello.txt 파일을 쓰기 모드(w)로 열기
    file.writelines(txt)

"""

"""
def to_gif(images):
    converted_images = images.astype(np.uint8)
    imageio.mimsave("animation.gif", converted_images, fps=10)
    return embed.embed_file("animation.gif")

print(f"Frame features in train set: {train_data[0].shape}")
print(f"Frame masks in train set: {train_data[1].shape}")

_, sequence_model = run_experiment()   #학습

test_video = np.random.choice(test_df["video_name"].values.tolist())
print(f"Test video path: {test_video}")
test_frames = sequence_prediction(test_video)  #테스트
#to_gif(test_frames[:MAX_SEQ_LENGTH])

"""


from kafka import KafkaProducer
import os

#9092:32452/TCP
producer = KafkaProducer(bootstrap_servers=os.environ.get('KAFKA_HOST', 'node1:32452,node2:32452,node3:32452'))
print("------->>")
future = producer.send('test', key=b'my_key', value=b'my_value', partition=0)
result = future.get(timeout=100)
print(result)


from keras import layers
from keras import models
from keras import backend as K
from keras.optimizers import Adam
from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, CSVLogger
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
import os
import numpy as np
import pandas as pd

# GPU setting
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto(
            gpu_options = tf.GPUOptions(
                visible_device_list="2", # specify GPU number
                allow_growth=True)
        )
set_session(tf.Session(config=config))

K.set_image_data_format(data_format='channels_first')

def resblock(x, filter_num=64, kernel_size=(3, 3)):
    shortcut = x
    shortcut = layers.BatchNormalization()(shortcut)

    x = layers.Conv2D(filter_num, kernel_size, padding="same")(x)
    x = layers.BatchNormalization()(x)
    x = layers.Activation("relu")(x)
    x = layers.Conv2D(filter_num, kernel_size, padding="same")(x)
    x = layers.BatchNormalization()(x)
    x = layers.Add()([x, shortcut])
    x = layers.Activation("relu")(x)
    return x

def create_model(filter_num=64, block_num=4):
    inputs = layers.Input(shape=[1, 48, 48])
    x = inputs

    for i in range(block_num):
        if i > 0:
            x = layers.MaxPooling2D((2,2))(x)
            filter_num = filter_num * 2
        filter_num = [64, 128, 256, 64][i]
        x = layers.Conv2D(filter_num, (3,3), padding="same")(x)
        x = resblock(x, filter_num)
        x = resblock(x, filter_num)
        x = resblock(x, filter_num)
    x = layers.GlobalAveragePooling2D()(x)
    y = layers.Dense(8, activation="softmax")(x)

    model = models.Model(inputs=inputs, outputs=y)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

if __name__ == '__main__':

    X = np.load('facedataset.npy')
    label = pd.read_csv('fer2013new.csv')
    Y = label[label['Usage'] == 'Training']
    Y = Y.drop(['Usage', 'Image name', 'unknown', 'NF'], axis=1)
    del label

    X = (X.astype(np.float32) / 127.5) - 1

    X = X.reshape(-1, 1, 48, 48)

    X_train, X_valid, y_train, y_valid = train_test_split(X, Y, test_size=0.2, random_state=42)

    datagen = ImageDataGenerator(
        width_shift_range=0.2,  # 左右にずらす
        height_shift_range=0.2,  # 上下にずらす
        horizontal_flip=True  # 左右反転
    )

    baseSaveDir = './log/face20200110'
    checkpoint = os.path.join(baseSaveDir, 'classifier_{epoch:02d}epoch-loss{val_loss:.2f}-acc{val_acc:.2f}.hdf5')
    cp = ModelCheckpoint(filepath = checkpoint, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')
    csvlogger = CSVLogger('./log/face20200110/history_face_classifier.csv')

    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1)
    early_stopping = EarlyStopping(monitor='val_loss', patience=20, verbose=1)

    image_clf = create_model()
    image_clf.summary()

    history = image_clf.fit_generator(datagen.flow(X_train, y_train, batch_size=100),
                        steps_per_epoch=X_train.shape[0] // 100, epochs=1000,
                        callbacks=[reduce_lr, early_stopping, cp, csvlogger],
                        validation_data=(X_valid, y_valid))


from django.db import models
from django.contrib.auth.models import AbstractUser

class CustomUser(AbstractUser):
	last_request = models.DateTimeField(null=True, blank=True, default=None)

class SimplePost(models.Model):
	author = models.ForeignKey(CustomUser, on_delete=models.CASCADE, null=True, blank=True, related_name='own_posts')
	title = models.CharField(max_length=255, null=True, blank=True)
	content = models.TextField(null=True, blank=True)
	created = models.DateTimeField(auto_now_add=True)

class Like(models.Model):
	author = models.ForeignKey(CustomUser, on_delete=models.CASCADE, null=True, blank=True, related_name='own_likes')
	post = models.ForeignKey(SimplePost, on_delete=models.CASCADE, null=True, blank=True, related_name='post_likes')
	created = models.DateField(auto_now_add=True, blank=True, null=True)

	class Meta:
		unique_together = ['author', 'post']


from selenium import webdriver
import time
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.by import By

with open('job_scraping_multipe_pages.csv', 'w') as file:
    file.write("Job_title; Company; Salary; Location; Job_description \n")

driver=webdriver.Chrome()

# driver.get('https://www.jobsite.co.uk/')

driver.get('https://www.jobsite.co.uk/jobs/data-engineer/in-united-kingdom?radius=30') # The particular url I wish to scrap

driver.maximize_window()
time.sleep(5)

# //*[@id="ccmgt_explicit_accept"]/span

cookie= driver.find_element(By.XPATH, '//*[@id="ccmgt_explicit_accept"]/span') # Locate the button to accept cookies
try:
    cookie.click()
except:
    pass

time.sleep(5)
# job_title=driver.find_element(By.ID, 'keywords')
# job_title.click()
# job_title.send_keys('Software Engineer')
# time.sleep(1)

# location=driver.find_element(By.ID, 'location')
# location.click()
# location.send_keys('Manchester')
# time.sleep(1)

# dropdown=driver.find_element(By.ID, 'Radius')
# radius=Select(dropdown)
# radius.select_by_visible_text('30 miles')
# time.sleep(1)

# search=driver.find_element(By.ID, 'search-button')
# search.click()
# time.sleep(50)

for k in range(5):
    titles=driver.find_elements(By.XPATH, '//div[@class="resultlist-dpthza"]')
    company=driver.find_elements(By.XPATH, '//div[@class="resultlist-1mjyynf"]')
    salary=driver.find_elements(By.XPATH, '//div[@class="resultlist-noiqwp"]/span')
    location=driver.find_elements(By.XPATH, '//div[@class="resultlist-5dwl9c"]')
    job_details=driver.find_elements(By.XPATH, '//div[@class="resultlist-1n63zbq"]')

    with open('job_scraping_multipe_pages.csv', 'a') as file:
        for i in range(len(titles)):
            file.write(titles[i].text + ";" + company[i].text + ";" + salary[i].text + ";" + location[i].text + ";"+
                      job_details[i].text + "\n")

        next=driver.find_element(By.XPATH, '//a[@aria-label="Next"]')
        next.click()
        time.sleep(7)
    file.close()
driver.close()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

# name: Zhornichenko Ilya Alekseevich
# group: TMSS
# course: Image analysis

# Importing libs

import numpy as np
from PIL import Image
from PIL import ImageChops
import scipy as sc
import matplotlib.pyplot as plt
from scipy.fftpack import fft, fftshift, ifft
import random

#correectly resampled image
def padImage(img):
    N0, N1 = img.size
    lenDiag = int(np.ceil(np.sqrt(N0**2+N1**2)))
    imgPad = Image.new('L',(lenDiag, lenDiag))
    c0, c1 = int(round((lenDiag-N0)/2)), int(round((lenDiag-N1)/2))
    imgPad.paste(img, (c0,c1))
    return imgPad, c0, c1
# getting sinogramm
def getProj(img, theta):
    numAngles = len(theta)
    sinogram = np.zeros((img.size[0],numAngles))
    for n in range(numAngles):
        rotImgObj = img.rotate(90-theta[n], resample=Image.BICUBIC)
        sinogram[:,n] = np.sum(rotImgObj, axis=0)
    return sinogram

#ramp filter for sinogram
def rmp_filter(sino):
    a = 0.1
    projLen, numAngles = sino.shape
    step = 2*np.pi/projLen
    w = np.arange(-np.pi, np.pi, step)
    if len(w) < projLen:
        w = np.concatenate([w, [w[-1]+step]])

    rn1 = abs(2/a*np.sin(a*w/2));
    rn2 = np.sin(a*w/2)/(a*w/2);
    r = rn1*(rn2)**2;

    filt = fftshift(r)
    filtered_sinogram = np.zeros((projLen, numAngles))
    for i in range(numAngles):
        projfft = fft(sino[:,i])
        filtProj = projfft*filt
        filtered_sinogram[:,i] = np.real(ifft(filtProj))

    return filtered_sinogram
#fbp reconstruction algoritm
def backproject(sinogram, theta):

    imageLen = sinogram.shape[0]
    reconMatrix = np.zeros((imageLen, imageLen))

    x = np.arange(imageLen)-imageLen/2
    y = x.copy()
    X, Y = np.meshgrid(x, y)

    plt.ion()
    fig2, ax = plt.subplots()
    im = plt.imshow(reconMatrix, cmap='gray')

    theta = theta*np.pi/180
    numAngles = len(theta)

    for n in range(numAngles):
        Xrot = X*np.sin(theta[n])-Y*np.cos(theta[n])
        XrotCor = np.round(Xrot+imageLen/2)
        XrotCor = XrotCor.astype('int')
        projMatrix = np.zeros((imageLen, imageLen))
        m0, m1 = np.where((XrotCor >= 0) & (XrotCor <= (imageLen-1)))
        s = sinogram[:,n]
        projMatrix[m0, m1] = s[XrotCor[m0, m1]]
        reconMatrix += projMatrix
    plt.close()
    plt.ioff()
    backprojArray = np.flipud(reconMatrix)
    return backprojArray

def delimeter(ramp_filtered_sino, delimeter):
    ramp_filtered_sino[:,::delimeter] = 255
    return ramp_filtered_sino


from functools import reduce
from settings import INSTALLED_APPS

def get_server_actions():
    apps = list(map(
        lambda item: __import__(f'{item}.routes'),
        INSTALLED_APPS
    ))

    routes = list(map(
        lambda item: getattr(item, 'routes', None),
        apps
    ))

    return reduce(
        lambda value, item: value + getattr(item, 'actionmapping', None),
        routes,
        []
    )

def resolve(action):
    actionmapping = {
        item.get('action'): item.get('controller')
        for item in get_server_actions()
        if item
    }

    return actionmapping.get(action)


from transformers import BertTokenizer, BertForSequenceClassification, BertModel
import torch
from torch import nn

class BertHumorNetwork(nn.Module):
    def __init__(self):
        super(BertHumorNetwork, self).__init__()
        model_name = "bert-base-uncased"
        self.bert = BertModel.from_pretrained(model_name, num_labels=2)

        self.linear1 = nn.Linear(768, 256)
        self.relu = nn.LeakyReLU(0.3)
        self.linear2 = nn.Linear(256, 1)

        self.sigmoid = nn.Sigmoid()

    def forward(self, ids, mask):
        output = self.bert(ids, mask)
        # print("AAA")
        # print(output[0].shape)

        # sequence_output has the following shape: (batch_size, sequence_length, 768)
        linear1_output = self.linear1(output[0][:, 0, :].view(-1, 768))   # extract the 1st token's embeddings

        relu_output = self.relu(linear1_output)

        linear2_output = self.linear2(relu_output)

        sigmoid_output = 3 * self.sigmoid(linear2_output)

        sigmoid_output = sigmoid_output.squeeze()

        return sigmoid_output.float()

class MixedBertHumorNetwork(nn.Module):
    def __init__(self):
        super(MixedBertHumorNetwork, self).__init__()
        model_name = "bert-base-uncased"
        self.bert = BertModel.from_pretrained(model_name, num_labels=2)

        self.bertlinear1 = nn.Linear(768, 256)

        self.dropout = nn.Dropout()

        self.relu = nn.LeakyReLU(0.3)

        self.baseline_linear1 = nn.Linear(12, 12)

        self.linear2 = nn.Linear(256 + 12, 1)

        self.sigmoid = nn.Sigmoid()

        for param in self.bert.parameters():
            param.requires_grad = False

        self.bertlinear1.requires_grad = True

        self.baseline_linear1.requires_grad = True

        self.linear2.requires_grad = True

    def forward(self, ids, mask, baseline_vec):
        baseline_vec = baseline_vec / torch.norm(baseline_vec)
        output = self.bert(ids, mask)
        # print("AAA")
        # print(output[0].shape)

        # sequence_output has the following shape: (batch_size, sequence_length, 768)
        bert_linear1_output = self.bertlinear1(output[0][:, 0, :].view(-1, 768))   # extract the 1st token's embeddings

        bert_linear1_output = self.dropout(bert_linear1_output)

        relu_bert_output = self.relu(bert_linear1_output)

        baseline_linear_output = self.baseline_linear1(baseline_vec)

        baseline_linear_output = self.dropout(baseline_linear_output)

        relu_baseline_output = self.relu(baseline_linear_output)

        cat_vec = torch.cat((relu_baseline_output, relu_bert_output), 1)

        linear2_output = self.linear2(cat_vec)

        sigmoid_output = 3 * self.sigmoid(linear2_output)

        sigmoid_output = sigmoid_output.squeeze()

        return sigmoid_output.float()


import cv2
import torch
import h5py as  H
import numpy as np
import scipy.io as sio
import img as I
from matplotlib import pyplot as plt
import torch.utils.data as data
import torchvision.transforms.functional as F

class MPII(data.Dataset):
	def __init__(self, split):
		print('==> initializing 2D {} data.'.format(split))
		self.split = split
		self.maxScale = 1
		self.inputRes = 256
		self.outputRes = 256
		self.nJoints = 16
		self.hmGauss = 2
		tags = ['imgname','part','center','scale']

		# self.stuff1 = sio.loadmat(open(opts.worldCoors[:-4] + ('train' if split is 'train' else '') + '.mat', 'rb'))['a']
		# self.stuff2 = sio.loadmat(open(opts.headSize[:-4] + ('train' if split is 'train' else '') + '.mat', 'rb'))['headSize']

		f = H.File('/home/rohit/Adversarial-Pose-Estimation/mpii/pureannot/{}.h5'.format(split), 'r')
		annot = {}
		for tag in tags:
			annot[tag] = np.asarray(f[tag]).copy()
		f.close()
		self.annot = annot

		self.len = len(self.annot['scale'])
		print('Loaded 2D {} {} samples'.format(split, len(annot['scale'])))

	def LoadImage(self, index):
		# print(self.annot['imgname'][index])
		# path = '/home/rohit/Adversarial-Pose-Estimation/mpii/images/{}'.format(''.join(chr(int(i)) for i in self.annot['imgname'][index]))
		path = '/home/rohit/Adversarial-Pose-Estimation/mpii/images/{}'.format(self.annot['imgname'][index])
		img = cv2.imread(path)
		return img

	def GetPartInfo(self, index):
		pts = self.annot['part'][index].copy()
		c = self.annot['center'][index].copy()
		s = self.annot['scale'][index]
		s = s * 200
		return pts, c, s

	def __getitem__(self, index):
		img = self.LoadImage(index)
		pts, c, s = self.GetPartInfo(index)
		r = 0

		if self.split == 'train':
			# s = s * (2 ** I.Rnd(self.maxScale))
			s = s * 1
			r = 0

		inp = I.Crop(img, c, s, r, self.inputRes) / 255.
		out = np.zeros((self.nJoints, self.outputRes, self.outputRes))

		for i in range(self.nJoints):
			if pts[i][0] > 1:
				pts[i] = I.Transform(pts[i], c, s, r, self.outputRes)
				out[i] = I.DrawGaussian(out[i], pts[i], self.hmGauss, 0.5 if self.outputRes==32 else -1)

		if self.split == 'train':
			if np.random.random() < 0.5:
				inp = I.Flip(inp)
				out = I.ShuffleLR(I.Flip(out))
				pts[:, 0] = self.outputRes - pts[:, 0]

			inp[0] = np.clip(inp[0] * (np.random.random() * (0.4) + 0.6), 0, 1)
			inp[1] = np.clip(inp[1] * (np.random.random() * (0.4) + 0.6), 0, 1)
			inp[2] = np.clip(inp[2] * (np.random.random() * (0.4) + 0.6), 0, 1)

		return {
			'image': torch.Tensor(inp),
			'heatmaps': torch.Tensor(out),
			'occlusions': torch.Tensor(np.zeros(out.shape)),
		}
		# return inp, out
		# if self.opts.TargetType=='heatmap':
		# 	return inp, out#, self.stuff1[index], self.stuff2[index]
		# elif self.opts.TargetType=='direct':
		# 	return inp, np.reshape((pts/self.opts.outputRes), -1) - 0.5#, self.stuff1[index], self.stuff2[index]

	def __len__(self):
		return self.len

if __name__ == '__main__':
	dataset = MPII('train')
	for i in range(len(dataset)):
		ii = np.random.randint(len(dataset))
		data = dataset[ii]
		plt.subplot(1, 2, 1)
		plt.imshow(data['image'].transpose(1, 2, 0)[:, :, ::-1] + 0)
		plt.subplot(1, 2, 2)
		print(data['heatmaps'].shape)
		plt.imshow(data['heatmaps'].max(0))
		plt.show()


import itertools
import multiprocessing as mp
import os
import shutil
from functools import partial

import numpy as np
from astropy.table import Table

import pypetal.pyzdcf.utils as utils
from pypetal.pyzdcf.plotting import plot_pyzdcf_results
from pypetal.utils import defaults

#For multiprocessing
def mp_map(func, args, threads):
    n_inputs = len(args)

    if (threads > 1) & (n_inputs > 1):
        pool = mp.get_context('fork').Pool(threads)
        res = pool.starmap(func, args)

        pool.close()
        pool.join()

    else:
        res = list(itertools.starmap(func, args))

    return res

def pyzdcf_tot(cont_fname, line_fnames, line_names, output_dir,
               general_kwargs, kwargs):

    if line_fnames is str:
        line_fnames = [line_fnames]
        line_names = [line_names]

    #--------------------------------------------------
    #Read general kwargs

    verbose = general_kwargs['verbose']
    plot = general_kwargs['plot']
    time_unit = general_kwargs['time_unit']
    lc_unit = general_kwargs['lc_unit']
    lag_bounds = general_kwargs['lag_bounds']
    threads = general_kwargs['threads']

    #--------------------------------------------------
    #Read kwargs

    nsim, minpts, uniform_sampling, omit_zero_lags, \
        sparse, prefix, run_plike, plike_dir = defaults.set_pyzdcf(kwargs,
                                                                   np.hstack([ [cont_fname], line_fnames ])
                                                                   )

    #-------------------------------------------
    if (run_plike) & (plike_dir is None):
        print('Error: plike_dir must be specified if run_plike=True')
        print('Skipping PLIKE')
        run_plike = False

    input_dir = os.path.dirname( os.path.realpath(cont_fname) )

    for i in range(len(line_fnames)):
        if input_dir != os.path.dirname( os.path.realpath(line_fnames[i]) ):
            print('ERROR: All light curve files must be in the same directory')
            return {}

    if verbose:

        txt_str = """
Running pyZDCF
----------------------
nsim: {}
minpts: {}
uniform_sampling: {}
omit_zero_lags: {}
sparse: {}
prefix: {}
run_plike: {}
plike_dir: {}
----------------------
        """.format( nsim, minpts, uniform_sampling, omit_zero_lags,
                    sparse, prefix, run_plike, plike_dir)

        print(txt_str)

    input_dir += r'/'
    cont_fname_short = os.path.basename(cont_fname)
    line_fnames_short = [ os.path.basename(x) for x in line_fnames ]

    pyzdcf_func = partial( utils.get_zdcf, num_MC=nsim, minpts=minpts,
                           uniform_sampling=uniform_sampling, omit_zero_lags=omit_zero_lags,
                           sparse=sparse, sep=',', verbose=False)

    arg1 = np.full( len(line_fnames), input_dir )
    arg2 = np.full( len(line_fnames), cont_fname_short )
    arg3 = line_fnames_short
    arg4 = [ output_dir + x + r'/pyzdcf/' for x in line_names[1:] ]
    arg5 = [ x + '_' + prefix for x in line_names[1:] ]

    args = list(zip(arg1, arg2, arg3, arg4, arg5))
    res_tot = mp_map(pyzdcf_func, args, threads)

    plike_tot = []
    for i in range(len(line_fnames)):

        x1, y1, yerr1 = np.loadtxt( cont_fname, delimiter=',', unpack=True, usecols=[0,1,2] )
        x2, y2, yerr2 = np.loadtxt( line_fnames[i], delimiter=',', unpack=True, usecols=[0,1,2] )

        plike_dict = None
        if run_plike:

            utils.run_plike( arg4[i] + arg5[i] + '.dcf', lag_bounds[i], plike_dir,
                            verbose=verbose)

            plike_fname = plike_dir + 'plike.out'
            output_fname = output_dir + line_names[i+1] + r'/pyzdcf/' + line_names[i+1] + '_plike.out'
            assert os.path.exists( os.path.dirname(output_fname) )

            shutil.move( plike_fname, output_fname )
            plike_dat = Table.read( output_fname,
                                    format='ascii',
                                    names=['num', 'lag', 'r', '-dr', '+dr', 'likelihood'])

            file = open(output_fname, 'r')
            output_str = list(file)[-3:]

            ml_lag = float( output_str[1].split()[7] )
            ml_lag_err_hi = np.abs( float( output_str[1].split()[8] )  )
            ml_lag_err_lo = np.abs( float( output_str[1].split()[9] )  )

            plike_dict = {
                'output': plike_dat,
                'ML_lag': ml_lag,
                'ML_lag_err_hi': ml_lag_err_hi,
                'ML_lag_err_lo': ml_lag_err_lo
            }

            plike_tot.append( plike_dict )

        plot_pyzdcf_results(x1, y1, yerr1, x2, y2, yerr2,
                                     res_tot[i], plike_dict,
                                     time_unit=time_unit, lc_unit=[lc_unit[0], lc_unit[i+1]],
                                     lc_names=[line_names[0], line_names[i+1]],
                                     fname=output_dir + line_names[i+1] + r'/pyzdcf/' + line_names[i+1] + '_zdcf.pdf',
                                     show=plot)

    return res_tot, plike_tot