from collections import deque,namedtuple
import networkx as nx

_LockRequest = namedtuple('_LockRequest', ['transaction', 'lock_type'])
_LockTableResponse = namedtuple('_LockTableResponse', ['response', 'transactions'])

class LockTable(object):
    """LockTable for a single site.

    Attributes
    ----------
    lock_table : dict of dicts
        The lock table is a dict, keyed by the variable names. Each
        variable name maps to three objects:
            RL : set
                A set of Transactions currently holding read-locks on x
            WL : Transaction or None
                Transaction currently holding (exclusive) write-lock on x
            waiting : deque
                A deque of _LockRequests, each with a transaction and the
                requested lock type
    waits_for : networkx.DiGraph
        A digraph with waits-for edges for locks at this specific site.
    """
    def __init__(self,variables):

        self.lock_table = {
            x: {
                'RL': set(),
                'WL': None,
                'waiting': deque()
            } for x in variables
        }

        self.waits_for = nx.DiGraph()

    def RL_available(self,T,x):
        """ Check if transaction T can get a read lock on x. A transaction
        can get a read lock on x if:
            - It already holds a read lock or a write lock on x
            - It does not already hold a sufficient lock, and no
              other transactions are either (i) currently holding a
              write lock, or (ii) have write-lock requests waiting in the
              queue.

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting RL on x
        x : Variable name
            Variable for which RL is requested

        Returns
        -------
        _LockTableResponse
            _LockTableResponse with two attributes:
                response : bool
                    True if RL available, False if not
                transactions : set
                    Set of transactions for which T would need to wait for RL.
                    Empty if lock is available.
        """
        holding_write_lock = self.lock_table[x]['WL']
        waiting_for_write_lock = set(other.transaction for other in self.lock_table[x]['waiting']
                                     if other.lock_type == 'WL').difference([T])

        # Case 1 -- already holding a sufficient lock
        if (holding_write_lock == T) or (T in self.lock_table[x]['RL']):
            return _LockTableResponse(response=True,
                                      transactions=set())

        # Case 2 -- not already holding a sufficient lock
        else:
            # Case 2a -- another transaction is currently holding a WL
            if (holding_write_lock is not None):
                return _LockTableResponse(
                    response=False,
                    transactions=set([holding_write_lock]).union(waiting_for_write_lock)
                )

            # Case 2b -- no other transaction is currently holding a write lock,
            # but some other transaction is waiting for a write lock
            elif (holding_write_lock is None) and (len(waiting_for_write_lock) > 0):
                return _LockTableResponse(
                    response=False,
                    transactions=waiting_for_write_lock
                )

            # Case 2c -- all other cases, T can have lock
            else:
                return _LockTableResponse(response=True,
                                          transactions=set())

    def WL_available(self,T,x):
        """ Check if transaction T can get write lock on x. A transaction can get a write
        lock on x if
            - It already holds a write-lock on x
            - It is the only transaction holding a read-lock on x (implying no T' has a WL),
              and there are also no other transactions waiting in the queue for a lock on x.
              Note it would be possible for T to hold a RL on x, and thus create a queue
              (if there is a T' requesting a WL, followed by any sequence of read
              or write lock requests)
            - No transactions are holding either a read or write lock on x and no
              transactions are waiting on a lock.
        In any other case, (i) some transaction has a lock on x, and/or (ii) other
        transactions are waiting for a lock -- so this lock request waits for
        all other queued transactions.

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting WL on x
        x : Variable name
            Variable for which WL is requested

        Returns
        -------
        _LockTableResponse
            _LockTableResponse with two attributes:
                response : bool
                    True if WL available, False if not
                transactions : set
                    Set of transactions for which T would need to wait for WL.
                    Empty if lock is available.
        """

        waiting_for_lock = set(other.transaction for other in self.lock_table[x]['waiting']).difference([T])

        # Case 1 -- T already has WL
        if (self.lock_table[x]['WL'] == T):
            return _LockTableResponse(response=True,
                                     transactions=set())

        # Case 2 -- T is the only transaction with a RL and no T' are waiting for locks
        if ((T in self.lock_table[x]['RL']) and
            (len(self.lock_table[x]['RL']) == 1) and
            (len(waiting_for_lock) == 0)):
            return _LockTableResponse(response=True,
                                     transactions=set())

        # Case 3 -- no locks held and no waiters
        if ((self.lock_table[x]['WL'] is None) and
            (len(self.lock_table[x]['RL']) == 0) and
            (len(waiting_for_lock) == 0)):
            return _LockTableResponse(response=True,
                                     transactions=set())

        # All other cases, T needs to wait for any non-T transaction that is
        # currently holding or waiting for a lock on x

        waiting_for = set()

        # Waits for transaction holding write lock
        holding_write_lock = self.lock_table[x]['WL']
        if (holding_write_lock is not None):
            waiting_for = set([holding_write_lock])

        # Waits for any other transaction currently holding RL on x
        other_transactions_w_RLs = self.lock_table[x]['RL'].difference([T])
        if other_transactions_w_RLs is not None:
            waiting_for = waiting_for.union(other_transactions_w_RLs)

        # Waits for any other transactions waiting for a lock
        if waiting_for_lock is not None:
            waiting_for = waiting_for.union(waiting_for_lock)

        return _LockTableResponse(
            response=False,
            transactions=waiting_for
        )

    def give_transaction_WL(self,T,x):
        """ Gives transaction T a WL on x.

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting WL on x
        x : Variable name
            Variable for which WL is requested

        Notes
        ------------
        - Updates both the LockTable, and the set of WL's held by T
        """
        self.lock_table[x]['WL'] = T
        T.write_locks[self].add(x)

    def request_write_lock(self,T,x):
        """Check if the transaction T is waiting on a WL
        on x at this site. If not, have T request a WL
        on this site, and give T this lock if it's available, or
        add it T to the queue if not. In either case, add
        this site to T's locks_needed set.

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting WL on x
        x : Variable name
            Variable for which WL is requested

        Notes
        ------------
        - Updates both the lock table, and either T's locks_needed
          or the locks it is holding.
        """
        if x not in T.locks_needed[self]:
            # If not, then we need to try to give T the lock, or
            # add it to the queue, and update T's lock_needed dict
            T.locks_needed[self].add(x)
            lock_available, waiting_for = self.WL_available(T,x)
            if lock_available:
                self.give_transaction_WL(T,x)
            else:
                self.add_transaction_to_lock_queue(T,x,waiting_for,'WL')

    def give_transaction_RL(self,T,x):
        """ Gives transaction T a RL on x.

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting RL on x
        x : Variable name
            Variable for which RL is requested

        Notes
        ------------
        - Updates both the LockTable, and the set of RL's held by T
        """
        self.lock_table[x]['RL'].add(T)
        T.read_locks[self].add(x)

    def add_transaction_to_lock_queue(self,T,x,waiting_for,lock_type):
        """ Adds transaction T to the queue for a lock on variable x,
        and updates the waits-for graph.

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting lock on x
        x : Variable name
            Variable for which lock is requested
        waiting_for : set
            Set of transactions blocking T's request for the x lock
        lock_type : str (RL or WL)
            Type of lock requested

        Notes
        -----
        - Adds the lock on x at this site to the set of locks T
          needs to proceed.
        """
        self.lock_table[x]['waiting'].append(_LockRequest(transaction=T,lock_type=lock_type))
        T.locks_needed[self].add(x)
        for other in waiting_for:
            self.waits_for.add_edge(T,other)

    def reassign_locks(self,x):
        """
        Reassign locks (as needed) for variable x. Called after a
        Transaction is ended (committed or aborted), when locks are released.

        Parameters
        ----------
        x : Variable name
            Variable to reassign locks for

        Notes
        ------------
        - If locks reassigned, updates the locks held by transactions
          given new locks
        """
        gave_new_lock = True
        waiters = self.lock_table[x]['waiting']
        while gave_new_lock and (len(waiters)>0):
            next_in_line = self.lock_table[x]['waiting'].popleft()
            next_lock_type = next_in_line.lock_type
            requesting_T = next_in_line.transaction

            if (next_lock_type == 'RL'):
                # Check if RL available
                # Case 1: No transaction holds a WL
                if (self.lock_table[x]['WL'] is None):
                    self.give_transaction_RL(requesting_T,x)
                # Case 2: some other transaction has WL
                else:
                    gave_new_lock = False
                    # Put this transaction back at front of queue
                    self.lock_table[x]['waiting'].appendleft(next_in_line)

            # Requesting WL
            else:
                # Case 1: No other transaction holds a WL and
                # no other transactions have read locks
                if ((self.lock_table[x]['WL'] is None) and
                    (len(self.lock_table[x]['RL'].difference([requesting_T]))==0)):
                    self.give_transaction_WL(requesting_T,x)
                # Case 2: All other cases can't give lock
                else:
                    gave_new_lock = False
                    # Put this transaction back at front of queue
                    self.lock_table[x]['waiting'].appendleft(next_in_line)

    def remove_T_from_lock_table_and_waitsfor(self,T):
        """
        Completely removes T from the lock table and waits
        for graph, and updates the lock table (reassigning locks).

        Parameters
        ----------
        T : ReadWriteTransaction
            Transaction requesting lock

        Notes
        ------------
        - Remove T from the lock table
        - Remove T from the waits_for graph
        - Reassigns locks (where there were blocked transactions
          waiting on the lock)
        """
        for x,row in self.lock_table.items():
            # Remove T from the RL set if it is present
            row['RL'].discard(T)
            # Reset the WL entry to None if T was holding WL
            if row['WL'] == T:
                row['WL'] = None
            # Remove T from the queue for locks on x
            new_wait_list = deque()
            for waiter in row['waiting']:
                if waiter.transaction != T:
                    new_wait_list.append(waiter)
            row['waiting'] = new_wait_list

            # Then reassign locks
            self.reassign_locks(x)

        # Finally remove node from waits_for graph
        if T in self.waits_for.nodes():
            self.waits_for.remove_node(T)


import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, inspect

import numpy as np
import pandas as pd
import datetime as dt

from flask import Flask, jsonify

#################################################
# Database Setup
#################################################
engine = create_engine("sqlite:///Resources/hawaii.sqlite")

# reflect an existing database into a new model
Base = automap_base()
# reflect the tables
Base.prepare(engine, reflect=True)

# Save reference to the table
Measurement = Base.classes.measurement
Station = Base.classes.station

# Create our session (link) from Python to the DB
session = Session(engine)

#################################################
# Flask Setup
#################################################
app = Flask(__name__)

#################################################
# Flask Routes
#################################################

@app.route("/")
def index():
    """List all available api routes."""
    return (
        f"Available Routes:
"
        f"/api/v1.0/precipitation
"
        f" - list of rain totals from all stations
"
        f"/api/v1.0/stations
"
        f"- list of Station numbers and names
"
        f"/api/v1.0/tobs
"
        f"- list of prior year temperatures from all stations
"
        f"/api/v1.0/STARTTIME
"
        f"- When given the start date (yyyy-mm-dd), calculates the min/ave/max temp for all dates greate than and equal to teh start date.
"
        f"/api/v1.0/STARTTIME/ENDTIME
"
        f"- Please provide the start date (yyyy-mm-dd) and end date (yyyy-mm-dd)"
        )

@app.route("/api/v1.0/precipitation")
def percipitation():
    """Return percipitation"""
    # Query all date and prcp from Measurement
    results = session.query(Measurement).all()
    # Create a dictionary from the row data and append to a list of all_stations
    all_stations = []

    for result in results:
        stations_dict = {}
        stations_dict["date"] = result.date
        stations_dict["prcp"] = result.prcp
        all_stations.append(stations_dict)

    return jsonify(all_stations)

@app.route("/api/v1.0/stations")
def stations():
    """Return Station Names"""
    results = session.query(Station.station, Station.name,).all()
    # Create a dictionary from the row data and append to a list of all_stations
    all_stations=[]
    for result in results:
        stations_dict = {}
        stations_dict["station"] = result[0]
        stations_dict["name"] = result[1]
        all_stations.append(stations_dict)

    return jsonify(all_stations)

@app.route("/api/v1.0/tobs")
def tobs():
    #query for the dates and temps froma year fromt the last data point.
    query_date = dt.date(2017,8,23) - dt.timedelta(weeks=52)
    #create all data from 2016/8/23-2017/8/23
    lasttwelvemonths = (session.query(Measurement)
                   .filter(func.strftime(Measurement.date)>=query_date)
                   .order_by(Measurement.date)
                   .all())
    one_year_tobs=[]

    for last in lasttwelvemonths:
        tobs_dict = {}
        tobs_dict["date"] = last.date
        tobs_dict["tobs"] = last.tobs
        one_year_tobs.append(tobs_dict)

    return jsonify(one_year_tobs)

@app.route("/api/v1.0/<start>")
def trip1(start):

    trip_data = session.query(func.min(Measurement.tobs), func.avg(Measurement.tobs),
    func.max(Measurement.tobs)).filter(Measurement.date>=start).all()

    trip = list(np.ravel(trip_data))

    return jsonify(trip)

@app.route("/api/v1.0/<start>/<end>")
def trip2(start,end):

    trip_data = session.query(func.min(Measurement.tobs), func.avg(Measurement.tobs),
    func.max(Measurement.tobs)).filter(Measurement.date>=start).filter(Measurement.date<=end).all()

    trip = list(np.ravel(trip_data))

    return jsonify(trip)

if __name__ == '__main__':
    app.run(debug=True)


from __future__ import print_function
import numpy as np
import random

from rnn_layers import *

import tensorflow as tf
import time, os, json, sys
import numpy as np
import matplotlib.pyplot as plt

#import pandas as pd

import tensorflow.python.platform
from tensorflow.contrib import rnn
from tensorflow.contrib import legacy_seq2seq

import argparse

from six.moves import cPickle
from six import text_type

# this module is from the utils.py file of this folder
# it handles loading texts to digits (aka. tokens) which are recognizable for the model
from utils import TextLoader

from coco_utils import *
from captioning import *

# this module is from the char_rnn.py file of this folder
# the task is implementing the CharRNN inside the class definition from this file
from char_rnn import Model

from optparse import OptionParser

parser = OptionParser()
parser.add_option('-g', '--gpu_num', dest='gpu_num', type='string', default='0')

def part_1():
    # implement rnn_step_forward
    # errors should be less than 1e-8
    x, prev_h, Wx, Wh, b, expected_next_h = getdata_rnn_step_forward()

    next_h, _ = rnn_step_forward(x, prev_h, Wx, Wh, b)

    print('next_h error: ', rel_error(expected_next_h, next_h))

    ##############################################################################

    # implement rnn_step_backward
    # errors should be less than 1e-8
    np.random.seed(2177)

    x, h, Wx, Wh, b, expected_next_h = getdata_rnn_step_forward()
    out, cache = rnn_step_forward(x, h, Wx, Wh, b)
    dnext_h = np.random.randn(*out.shape)
    dx_num, dprev_h_num, dWx_num, dWh_num, db_num = getdata_rnn_step_backward(x,h,Wx,Wh,b,dnext_h)

    dx, dprev_h, dWx, dWh, db = rnn_step_backward(dnext_h, cache)

    print('dx error     : ', rel_error(dx_num, dx))
    print('dprev_h error: ', rel_error(dprev_h_num, dprev_h))
    print('dWx error    : ', rel_error(dWx_num, dWx))
    print('dWh error    : ', rel_error(dWh_num, dWh))
    print('db error     : ', rel_error(db_num, db))

    ##############################################################################

    # implement rnn_forward
    # errors should be less than 1e-7
    x,h0,Wx,Wh,b, expected_h = getdata_rnn_forward()

    h, _ = rnn_forward(x, h0, Wx, Wh, b)

    print('h error: ', rel_error(expected_h, h))

    ##############################################################################

    # implement rnn_forward
    # errors should be less than 1e-7
    np.random.seed(2177)

    x,h0,Wx,Wh,b, expected_h = getdata_rnn_forward()
    out, cache = rnn_forward(x, h0, Wx, Wh, b)
    dout = np.random.randn(*out.shape)
    dx_num, dh0_num, dWx_num, dWh_num, db_num = getdata_rnn_backward(x,h0,Wx,Wh,b,dout)

    dx, dh0, dWx, dWh, db = rnn_backward(dout, cache)

    print('dx error: ', rel_error(dx_num, dx))
    print('dh0 error: ', rel_error(dh0_num, dh0))
    print('dWx error: ', rel_error(dWx_num, dWx))
    print('dWh error: ', rel_error(dWh_num, dWh))
    print('db error: ', rel_error(db_num, db))

def part_2(gpu_num):

    model_path='./models_captioning'
    data_path ='./coco/coco_captioning'

    data = load_coco_data(base_dir=data_path)
    if len(data)==8 :
        print('COCO data load complete')

    for k, v in data.items():
        if type(v) == np.ndarray:
            print(k, type(v), v.shape, v.dtype)
        else:
            print(k, type(v), len(v))

#train_data = load_coco_data(base_dir=data_path, max_train=512*100)
    train_data = data

    captions = train_data['train_captions']
    img_idx  = train_data['train_image_idxs']
    img_features = train_data['features'][img_idx]
    word_to_idx = train_data['word_to_idx']
    idx_to_word = train_data['idx_to_word']

    n_words = len(word_to_idx)
    maxlen = train_data['train_captions'].shape[1]
    input_dimension = train_data['features'].shape[1]

    print(n_words)
    print(maxlen)
    print(img_features.shape, captions.shape)

    vcaptions = train_data['val_captions']
    vimg_idx  = train_data['val_image_idxs']
    vimg_features = train_data['features'][vimg_idx]
    print(vimg_features.shape, vcaptions.shape)

    def train(img_features, captions):
        #---------------------------------------------------------
        # You must show maxlen, n_words, caption.shap
        #---------------------------------------------------------
        print(maxlen, n_words, captions.shape)

        #################################################
        # TODO: Implement caption training
        # - save your trained model in model_path
        # - must print about 10 loss changes !!
        #################################################
        img_feature_dim = img_features.shape[1]
        batch_size = 512
        n_epochs = 100

        conf = tf.ConfigProto()
        conf.gpu_options.per_process_gpu_memory_fraction = 0.2
        os.environ['CUDA_VISIBLE_DEVICES'] = gpu_num

        def calc_caption_length(caption):
            for i, word_idx in enumerate(caption):
                if word_idx == 0:
                    return min(len(caption), i+2)
            return len(caption)
        caption_lens = np.array([calc_caption_length(caption) for caption in captions])
        vcaption_lens = np.array([calc_caption_length(caption) for caption in vcaptions])

        known_caption_idxs = np.all(captions != 3, axis=1)

        known_captions = captions[known_caption_idxs]
        known_caption_lens = caption_lens[known_caption_idxs]
        known_img_features = img_features[known_caption_idxs]

        tf.reset_default_graph()

        if gpu_num == '0':
            captioning = Captioning(img_feature_dim, maxlen, n_words, 386, 3)
        elif gpu_num == '1':
            captioning = Captioning(img_feature_dim, maxlen, n_words, 848, 3)
        elif gpu_num == '2':
            captioning = Captioning(img_feature_dim, maxlen, n_words, 496, 3)
        elif gpu_num == '3':
            captioning = Captioning(img_feature_dim, maxlen, n_words, 612, 3)
        input_sequences, input_sequence_lens, input_img_features, \
            loss, optimize, drop_out_keep_rate, teacher_force = captioning.build_model()
        predict_indices = captioning.predict()

        # Saver
        saved_model_path = 'saved_model_{}/ass3_show_and_tell.ckpt'.format(gpu_num)
        epoch_save_path = 'saved_model/ass3_epoch.txt'
        min_valid_loss = 9999.99
        max_BLEU = -1.0
        saver = tf.train.Saver()
        with tf.Session(config=conf) as sess:
            sess.run(tf.global_variables_initializer())

            def generate_minibatch():
                permutation = np.random.permutation(len(known_captions))
                return known_captions[permutation][:batch_size], \
                    known_caption_lens[permutation][:batch_size], \
                    known_img_features[permutation][:batch_size]

            def generate_minibatch_valid(data_size):
                permutation = np.random.permutation(data_size)
                return vcaptions[permutation][:batch_size], \
                    vcaption_lens[permutation][:batch_size], \
                    vimg_features[permutation][:batch_size]

            for epoch in range(n_epochs):
                start_time = time.time()
                data_size = captions.shape[0]
                steps = data_size // batch_size

                # train
                loss_sum = 0.0
                for step in range(steps):
                    input_seqs, input_seq_lens, input_imgs = generate_minibatch()
                    feed_dict = {
                        input_sequences: input_seqs,
                        input_sequence_lens: input_seq_lens,
                        input_img_features: input_imgs,
                        drop_out_keep_rate: 0.1,
                    }
                    train_loss, _, train_force = sess.run([loss, optimize, teacher_force], feed_dict=feed_dict)
                    loss_sum += train_loss

                # validation
                val_loss_sum = 0.0
                val_data_size = vcaptions.shape[0]
#                val_steps = val_data_size // batch_size
#                for step in range(val_steps):
#                    input_seqs, input_seq_lens, input_imgs = generate_minibatch_valid(val_data_size)
#                    feed_dict = {
#                        input_sequences: input_seqs,
#                        input_sequence_lens: input_seq_lens,
#                        input_img_features: input_imgs,
#                    }
#                    val_loss, val_force = sess.run([loss, teacher_force], feed_dict=feed_dict)
#                    val_loss_sum += val_loss

                # test sentence from validation
                pick_idx = random.randrange(val_data_size)
                feed_dict = {
                    input_img_features: np.expand_dims(img_features[pick_idx], axis=0),
                }

                pred_indices = sess.run([predict_indices], feed_dict=feed_dict)
                print([idx_to_word[idx] for idx in pred_indices[0][0]])

                # BLEU score
                def image_captioning(features) :
                    pr_captions = np.zeros((features.shape[0],maxlen),int)
                    feed_dict = {
                        input_img_features: features,
                    }
                    predicts = sess.run([predict_indices], feed_dict=feed_dict)
                    pr_captions = np.array(predicts[0])
                    return pr_captions

                def evaluate_model(data, split):
                    BLEUscores = {}

                    minibatch = sample_coco_minibatch(data, split=split, batch_size="All")
                    gt_captions, features, urls = minibatch
                    gt_captions = decode_captions(gt_captions, data['idx_to_word'])

                    pr_captions = image_captioning(features)
                    pr_captions = decode_captions(pr_captions, data['idx_to_word'])

                    total_score = 0.0

                    for gt_caption, pr_caption, url in zip(gt_captions, pr_captions, urls):
                        total_score += BLEU_score(gt_caption, pr_caption)

                    BLEUscores[split] = total_score / len(pr_captions)

                    return BLEUscores[split]

                cur_BLEU = evaluate_model(train_data, 'val')

                print('epoch {} : '.format(epoch) + ('train loss %.4f, ' % (loss_sum / steps)) + \
                        ('BLEU score %.4f' % cur_BLEU))

                if cur_BLEU > max_BLEU:
                    max_BLEU = cur_BLEU
                    saver.save(sess, saved_model_path)
                    print('model saved!!')

                if (epoch > 4) and (max_BLEU * 0.9) > cur_BLEU:
                    print('early stop!!')
                    break

        #print("Current Cost: ", loss_value, "\t Epoch {}/{}".format(epoch, n_epochs))

        #################################################
        #                END OF YOUR CODE               #
        #################################################

    train(img_features, captions)

def part_3(gpu_num):
    # for TensorFlow vram efficiency: if this is not specified, the model hogs all the VRAM even if it's not necessary
    # bad & greedy TF! but it has a reason for this design choice FWIW, try googling it if interested
    conf = tf.ConfigProto()
    conf.gpu_options.allow_growth = True
    os.environ['CUDA_VISIBLE_DEVICES'] = gpu_num

    # argparsing
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # Data and model checkpoints directories
    parser.add_argument('--data_dir', type=str, default='data/tinyshakespeare',
                                help='data directory containing input.txt with training examples')
    parser.add_argument('--save_dir', type=str, default='models_char_rnn',
                                help='directory to store checkpointed models')
    parser.add_argument('--save_every', type=int, default=1000,
                                help='Save frequency. Number of passes between checkpoints of the model.')
    parser.add_argument('--init_from', type=str, default=None,
                                help="""continue training from saved model at this path (usually "save").
                                     Path must contain files saved by previous training process:
                                     'config.pkl'        : configuration;
                                     'chars_vocab.pkl'   : vocabulary definitions;
                                     'checkpoint'        : paths to model file(s) (created by tf).
                                     Note: this file contains absolute paths, be careful when moving files around;
                                     'model.ckpt-*'      : file(s) with model definition (created by tf)
                                     Model params must be the same between multiple runs (model, rnn_size, num_layers and seq_length).
                                     """)
    # Model params
    parser.add_argument('--model', type=str, default='lstm',
                                help='lstm, rnn, gru, or nas')
    parser.add_argument('--rnn_size', type=int, default=128,
                                help='size of RNN hidden state')
    parser.add_argument('--num_layers', type=int, default=3,
                                help='number of layers in the RNN')
    # Optimization
    parser.add_argument('--seq_length', type=int, default=500,
                                help='RNN sequence length. Number of timesteps to unroll for.')
    parser.add_argument('--batch_size', type=int, default=128,
                                help="""minibatch size. Number of sequences propagated through the network in parallel.
                                        Pick batch-sizes to fully leverage the GPU (e.g. until the memory is filled up)
                                        commonly in the range 10-500.""")
    parser.add_argument('--num_epochs', type=int, default=50,
                                help='number of epochs. Number of full passes through the training examples.')
    parser.add_argument('--grad_clip', type=float, default=20.,
                                help='clip gradients at this value')
    parser.add_argument('--learning_rate', type=float, default=0.001,
                                help='learning rate')
    parser.add_argument('--decay_rate', type=float, default=0.97,
                                help='decay rate for rmsprop')
    parser.add_argument('--output_keep_prob', type=float, default=0.1,
                                help='probability of keeping weights in the hidden layer')
    parser.add_argument('--input_keep_prob', type=float, default=0.1,
                                help='probability of keeping weights in the input layer')

    # needed for argparsing within jupyter notebook
    # https://stackoverflow.com/questions/30656777/how-to-call-module-written-with-argparse-in-ipython-notebook
    sys.argv = ['-f']
    args = parser.parse_args()

    if gpu_num == '0':
        args.save_dir = 'models_char_rnn_{}'.format(gpu_num)
        args.init_from = args.save_dir
        args.rnn_size = 749
        args.num_layers = 1
        args.decay_rate = 0.5
        args.output_keep_prob = 0.1
        args.input_keep_prob = 0.1
        args.grad_clip = 5.0
        args.num_epochs = 500
    elif gpu_num == '1':
        args.save_dir = 'models_char_rnn_{}'.format(gpu_num)
        args.init_from = args.save_dir
        args.rnn_size = 486
        args.num_layers = 2
        args.decay_rate = 0.1
        args.output_keep_prob = 0.1
        args.input_keep_prob = 0.1
        args.grad_clip = 20.0
        args.num_epochs = 500
    elif gpu_num == '2':
        args.save_dir = 'models_char_rnn_{}'.format(gpu_num)
        args.init_from = args.save_dir
        args.rnn_size = 486
        args.num_layers = 2
        args.decay_rate = 0.1
        args.output_keep_prob = 0.1
        args.input_keep_prob = 0.1
        args.grad_clip = 10.0
        args.num_epochs = 500
    elif gpu_num == '3':
        args.save_dir = 'models_char_rnn_{}'.format(gpu_num)
        args.init_from = args.save_dir
        args.rnn_size = 749
        args.num_layers = 1
        args.decay_rate = 0.1
        args.output_keep_prob = 0.1
        args.input_keep_prob = 0.1
        args.grad_clip = 20.0
        args.num_epochs = 500

    # protip: always check the data and poke around the data yourself
    # you will get a lot of insights by looking at the data
    data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length)
    data_loader.reset_batch_pointer()

    # training loop definition
    def train(args):
        data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length)
        args.vocab_size = data_loader.vocab_size
        print("vocabulary size: " + str(args.vocab_size))

        # check compatibility if training is continued from previously saved model
        init_from = args.init_from
        if not os.path.isdir(args.init_from):
            init_from = None

        if init_from is not None:
            # check if all necessary files exist
            assert os.path.isdir(init_from)," %s must be a a path" % init_from
            assert os.path.isfile(os.path.join(init_from,"config.pkl")),"config.pkl file does not exist in path %s"%init_from
            assert os.path.isfile(os.path.join(init_from,"chars_vocab.pkl")),"chars_vocab.pkl.pkl file does not exist in path %s" % init_from
            ckpt = tf.train.latest_checkpoint(init_from)
            assert ckpt, "No checkpoint found"

            # open old config and check if models are compatible
            with open(os.path.join(init_from, 'config.pkl'), 'rb') as f:
                saved_model_args = cPickle.load(f)
            need_be_same = ["model", "rnn_size", "num_layers", "seq_length"]
            for checkme in need_be_same:
                assert vars(saved_model_args)[checkme]==vars(args)[checkme],"Command line argument and saved model disagree on '%s' "%checkme

            # open saved vocab/dict and check if vocabs/dicts are compatible
            with open(os.path.join(init_from, 'chars_vocab.pkl'), 'rb') as f:
                saved_chars, saved_vocab = cPickle.load(f)
            assert saved_chars==data_loader.chars, "Data and loaded model disagree on character set!"
            assert saved_vocab==data_loader.vocab, "Data and loaded model disagree on dictionary mappings!"

        if not os.path.isdir(args.save_dir):
            os.makedirs(args.save_dir)
        with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
            cPickle.dump(args, f)
        with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'wb') as f:
            cPickle.dump((data_loader.chars, data_loader.vocab), f)

        print("building the model... may take some time...")
        ##################### This line builds the CharRNN model defined in char_rnn.py #####################
        tf.reset_default_graph()
        model = Model(args)
        print("model built! starting training...")

        with tf.Session(config=conf) as sess:
            sess.run(tf.global_variables_initializer())
            saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
            # restore model
            if init_from is not None:
                saver.restore(sess, ckpt)
            for e in range(args.num_epochs):
                sess.run(tf.assign(model.lr, args.learning_rate * (args.decay_rate ** (e/args.num_epochs))))
                data_loader.reset_batch_pointer()
                state = sess.run(model.initial_state)

                for b in range(int(data_loader.num_batches)):
                    start = time.time()
                    x, y = data_loader.next_batch()
                    feed = {model.input_data: x, model.targets: y}
                    for i, (c, h) in enumerate(model.initial_state):
                        feed[c] = state[i].c
                        feed[h] = state[i].h

                    train_loss, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed)

                    end = time.time()

                    # print training log every 100 steps
                    if ((e * data_loader.num_batches + b) % 100 == 0):
                        print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
                                    .format(e * data_loader.num_batches + b,
                                    args.num_epochs * data_loader.num_batches,
                                    e, train_loss, end - start))
                    if (e * data_loader.num_batches + b) % args.save_every == 0\
                                    or (e == args.num_epochs-1 and b == data_loader.num_batches-1):
                        # save for the last result
                        checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
                        saver.save(sess, checkpoint_path, global_step=e * data_loader.num_batches + b)
                        print("model saved to {}".format(checkpoint_path))

    for i in range(1):
        train(args)
        print('==============================={}==================================='.format(i))
        part_3_sample(gpu_num)
        print('====================================================================')

def part_3_sample(gpu_num):
    conf = tf.ConfigProto()
    conf.gpu_options.allow_growth = True
    os.environ['CUDA_VISIBLE_DEVICES'] = gpu_num

    def sample_eval(args):
        with open(os.path.join(args.save_dir, 'config.pkl'), 'rb') as f:
            saved_args = cPickle.load(f)
        with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'rb') as f:
            chars, vocab = cPickle.load(f)
        #Use most frequent char if no prime is given
        if args.prime == '':
            args.prime = chars[0]
        tf.reset_default_graph()
        model = Model(saved_args, training=False)

        with tf.Session(config=conf) as sess:
            tf.global_variables_initializer().run()
            saver = tf.train.Saver(tf.global_variables())
            ckpt = tf.train.get_checkpoint_state(args.save_dir)
            if ckpt and ckpt.model_checkpoint_path:
                saver.restore(sess, ckpt.model_checkpoint_path)
                print(str(model.sample(sess, chars, vocab, args.n, args.prime)),'utf-8')

    parser_sample = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser_sample.add_argument('--save_dir', type=str, default='models_char_rnn',
            help='model directory to store checkpointed models')
    parser_sample.add_argument('-n', type=int, default=500,
            help='number of characters to sample')
    parser_sample.add_argument('--prime', type=text_type, default=u'',
            help='prime text')
    sys.argv = ['-f']

    args_sample = parser_sample.parse_args()
    args_sample.save_dir = 'models_char_rnn_{}'.format(gpu_num)

    sample_eval(args_sample)

def main():
    options, args = parser.parse_args()
    gpu_num = options.gpu_num

    part_3(gpu_num)
#part_3_sample(gpu_num)

if __name__ == '__main__':
    main()


import jsonlines
from pymongo import MongoClient
from os import listdir
from os.path import isfile, join
import argparse

# arguments for the script defined
parser = argparse.ArgumentParser(description='Import a folder content to MongoDB')
parser.add_argument('folder_name', type=str,
                    help='Folder name containing jsonlines files to parse, prefixed with the language code')

# open mongodb connection and collection
client = MongoClient('localhost', 27017)
db = client['references']
collection = 'all'

def get_files(folder_name):
    """ Get file names from a given folder
    :param folder_name: name of the folder that the jsonlines files are stored
    :return: list of file names
    """
    file_names = [f for f in listdir(folder_name) if isfile(join(folder_name, f))]
    return file_names

def load_file_data(folder_name, file_name):
    """ load data from jsonlines file
    Adds language code, assuming the file name is <languagecode>-references.ljson
    :param file_name: filename with data to be loaded into mongodb
    :return: list with data from the file
    """
    file_data = []
    with jsonlines.open(folder_name + '/' + file_name) as reader:
        for obj in reader:
            obj['language'] = file_name.split('-')[0]
            file_data.append(obj)
    return file_data

def insert_into_collection(file_data):
    """ Insert file_data into collection
    :param file_data: file data to be inserted into the collection
    :return:
    """
    collection_all = db[collection]
    # if pymongo < 3.0, use insert()
    #collection_english.insert(file_data)
    # if pymongo >= 3.0 use insert_one() for inserting one document
    #collection_english.insert_one(file_data)
    # if pymongo >= 3.0 use insert_many() for inserting many documents
    collection_all.insert_many(file_data)

if __name__ == '__main__':
    args = parser.parse_args()
    file_names = get_files(args.folder_name)
    for file_name in file_names:
        print('Starting to input the file: ' + str(file_name))
        file_data = load_file_data(args.folder_name, file_name)
        try:
            insert_into_collection(file_data)
        except Exception as e:
            print('Exception:')
            print(e)

    client.close()


import argparse
import os
import sys
import pickle
import time
sys.path.append(os.getcwd())

from khrylib.utils import *
from khrylib.rl.core.policy_mcp_gaussian import MCPPolicyGaussian
from khrylib.rl.core.policy_additive_gaussian import AdditivePolicyGaussian
from khrylib.rl.core.policy_split import SplitPolicyGaussian
from khrylib.rl.core.policy_cond_gaussian import CondPolicyGaussian
from khrylib.rl.core.critic import GoalValue
from khrylib.rl.agents import AgentPPO
from khrylib.models.mlp import MLP
from torch.utils.tensorboard import SummaryWriter
from motion_imitation.envs.humanoid_im import HumanoidEnv
#from motion_imitation.envs.humanoid_im_nimble import HumanoidNimbleEnv
from motion_imitation.utils.config import Config
from motion_imitation.reward_function import reward_func
from motion_imitation.expert_scheduler import ExpertScheduler

policy_dict = {"multiplicative":MCPPolicyGaussian,
               "additive":AdditivePolicyGaussian,
               "split": SplitPolicyGaussian,
               "cond": CondPolicyGaussian}

parser = argparse.ArgumentParser()
parser.add_argument('--cfg', default=None)
parser.add_argument('--render', action='store_true', default=False)
parser.add_argument('--test', action='store_true', default=False)
parser.add_argument('--num_threads', type=int, default=24)
parser.add_argument('--gpu_index', type=int, default=0)
parser.add_argument('--iter', type=int, default=0)
parser.add_argument('--policy',type=str, default="multiplicative")
parser.add_argument('--show_noise', action='store_true', default=False)
parser.add_argument('--change_goal_freq',type=int, default=3)
parser.add_argument('--simulator',type=str, default="mujoco")
parser.add_argument('--exp_name', type=str, default=None, required=True)
parser.add_argument('--load_name', type=str, default=None)
parser.add_argument('--load_iter', type=int, default=None)
parser.add_argument('--freeze', action="store_true", default=False)
parser.add_argument('--collect_data', action='store_true', default=False)
parser.add_argument('--use_wandb',action="store_true", default=False)
args = parser.parse_args()
if args.render:
    args.num_threads = 1

cfg = Config(args.cfg, args.test, create_dirs=not (args.render or args.iter > 0))

dtype = torch.float64
torch.set_default_dtype(dtype)
device = torch.device('cuda', index=args.gpu_index) if torch.cuda.is_available() else torch.device('cpu')
if torch.cuda.is_available():
    torch.cuda.set_device(args.gpu_index)
np.random.seed(cfg.seed)
torch.manual_seed(cfg.seed)
tb_logger = SummaryWriter(cfg.tb_dir) if not args.render else None
logger = create_logger(os.path.join(cfg.log_dir, 'log.txt'), file_handle=not args.render)

"""environment"""
if args.simulator == "mujoco":
    env = HumanoidEnv(cfg, filename=args.exp_name if args.collect_data else None)
elif args.simulator == "nimble":
    raise NotImplementedError
    env = HumanoidNimbleEnv(cfg)
env.seed(cfg.seed)
#actuators = env.model.actuator_names
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.shape[0]
running_state = ZFilter((state_dim,), clip=5)

exp_sch = ExpertScheduler(n_experts = env.get_num_expert(), expected_freq=args.change_goal_freq)
"""define actor and critic"""
#policy_net = PolicyGaussian(MLP(state_dim, cfg.policy_hsize, cfg.policy_htype), action_dim, log_std=cfg.log_std, fix_std=cfg.fix_std)
policy_net = policy_dict[args.policy](MLP(state_dim + (0 if args.policy!= 'cond' else 8), cfg.policy_hsize, cfg.policy_htype),
                                      action_dim, log_std=cfg.log_std, fix_std=cfg.fix_std, goal_dim=39)
if args.freeze:
    policy_net.freeze()
value_net = GoalValue(MLP(state_dim+39, cfg.value_hsize, cfg.value_htype)) # 39 is goal dim
policy_net.set_goal(env.get_goal())
value_net.set_goal(env.get_goal())
if args.load_iter is not None:
    cp_path = f"{cfg.model_dir}/iter_{args.load_name}_{args.load_iter:04}.p"
    logger.info('loading model from checkpoint: %s' % cp_path)
    model_cp = pickle.load(open(cp_path, "rb"))
    policy_net.load_state_dict(model_cp['policy_dict'])
    value_net.load_state_dict(model_cp['value_dict'])
    running_state = model_cp['running_state']
to_device(device, policy_net, value_net)

if cfg.policy_optimizer == 'Adam':
    optimizer_policy = torch.optim.Adam(policy_net.parameters(), lr=cfg.policy_lr, weight_decay=cfg.policy_weightdecay)
else:
    optimizer_policy = torch.optim.SGD(policy_net.parameters(), lr=cfg.policy_lr, momentum=cfg.policy_momentum, weight_decay=cfg.policy_weightdecay)
if cfg.value_optimizer == 'Adam':
    optimizer_value = torch.optim.Adam(value_net.parameters(), lr=cfg.value_lr, weight_decay=cfg.value_weightdecay)
else:
    optimizer_value = torch.optim.SGD(value_net.parameters(), lr=cfg.value_lr, momentum=cfg.value_momentum, weight_decay=cfg.value_weightdecay)

# reward functions
expert_reward = reward_func[cfg.reward_id]

"""create agent"""
agent = AgentPPO(env=env, dtype=dtype, device=device, running_state=running_state,
                 custom_reward=expert_reward, mean_action=args.render and not args.show_noise,
                 render=args.render, num_threads=args.num_threads,
                 policy_net=policy_net, value_net=value_net,
                 optimizer_policy=optimizer_policy, optimizer_value=optimizer_value, opt_num_epochs=cfg.num_optim_epoch,
                 gamma=cfg.gamma, tau=cfg.tau, clip_epsilon=cfg.clip_epsilon,
                 policy_grad_clip=[(policy_net.parameters(), 40)], end_reward=cfg.end_reward,
                 use_mini_batch=cfg.mini_batch_size < cfg.min_batch_size, mini_batch_size=cfg.mini_batch_size)

def pre_iter_update(i_iter):
    cfg.update_adaptive_params(i_iter)
    agent.set_noise_rate(cfg.adp_noise_rate)
    set_optimizer_lr(optimizer_policy, cfg.adp_policy_lr)
    if cfg.fix_std:
        if args.policy != "split":
            policy_net.action_log_std.fill_(cfg.adp_log_std)
        else:
            policy_net.action_log_std_lb.fill_(cfg.adp_log_std)
            policy_net.action_log_std_ub.fill_(cfg.adp_log_std)
    return

# Initialize wandb
if args.use_wandb:
    import wandb
    wandb.init()

def main_loop():

    if args.render:
        pre_iter_update(args.iter)
        agent.sample(1e8)
    else:
        average_length = np.zeros(agent.env.get_num_expert())
        for i_iter in range(args.iter, cfg.max_iter_num):
            """generate multiple trajectories that reach the minimum batch_size"""
            pre_iter_update(i_iter)
            batch, log = agent.sample(cfg.min_batch_size)
            if cfg.end_reward:
                agent.env.end_reward = log.avg_c_reward * cfg.gamma / (1 - cfg.gamma)

            """update networks"""
            t0 = time.time()
            agent.update_params(batch)
            t1 = time.time()

            """logging"""
            c_info = log.avg_c_info
            logger.info(
                '{}\tT_sample {:.2f}\tT_update {:.2f}\tETA {}\texpert_R_avg {:.4f} {}'
                '\texpert_R_range ({:.4f}, {:.4f})\teps_len {:.2f}'
                .format(i_iter, log.sample_time, t1 - t0, get_eta_str(i_iter, cfg.max_iter_num, t1 - t0 + log.sample_time), log.avg_c_reward,
                        np.array2string(c_info, formatter={'all': lambda x: '%.4f' % x}, separator=','),
                        log.min_c_reward, log.max_c_reward, log.avg_episode_len))
            if args.use_wandb:
                wandb.log({
                    "iter":i_iter,
                    "eps_len": log.avg_episode_len,
                    "expert_R_avg": log.avg_c_reward,
                    "goal":agent.env.get_expert_id()
                })

            tb_logger.add_scalar('total_reward', log.avg_c_reward, i_iter)
            tb_logger.add_scalar('episode_len', log.avg_episode_reward, i_iter)
            for i in range(c_info.shape[0]):
                tb_logger.add_scalar('reward_%d' % i, c_info[i], i_iter)
                tb_logger.add_scalar('eps_reward_%d' % i, log.avg_episode_c_info[i], i_iter)

            """ Update goal from time to time """
            # Adaptive scheduling.s
            exp_sch.record_eps_len(log.avg_episode_len, agent.env.get_expert_id())
            print("Now expert is: ", agent.env.get_expert_id())
            if exp_sch.update(agent.env.get_expert_id(),i_iter):
                agent.env.switch_expert()
                goal = agent.env.get_goal()
                agent.policy_net.set_goal(goal)
                agent.value_net.set_goal(goal)

            """ Save model from time to time """
            if cfg.save_model_interval > 0 and (i_iter+1) % cfg.save_model_interval == 0:
                tb_logger.flush()
                with to_cpu(policy_net, value_net):
                    cp_path = f"{cfg.model_dir}/iter_{args.exp_name}_{i_iter+1:04}.p"
                    model_cp = {'policy_dict': policy_net.state_dict(),
                                'value_dict': value_net.state_dict(),
                                'running_state': running_state}
                    pickle.dump(model_cp, open(cp_path, 'wb'))

            """clean up gpu memory"""
            torch.cuda.empty_cache()
        #env.save_data()
        logger.info('training done!')

main_loop()


from uuid import uuid4

from gofer.messaging import Queue, NotFound

N = 10

class Address(object):

    def __init__(self, address):
        self.address = address
        self.parts = address.split('/')

    @property
    def exchange(self):
        if len(self.parts) > 1:
            return self.parts[0]
        else:
            return ''

    @property
    def queue(self):
        return self.parts[-1]

    def __str__(self):
        return self.address

class Test(object):

    def __init__(self, url, adapter):
        self.url = url
        self.adapter = adapter

    def producer_reader(self, address):
        print('using producer/reader')
        with self.adapter.Sender(url=self.url) as p:
            for x in range(0, N):
                print('#{} - sent: {}'.format(x, address))
                p.send(str(address), 'hello')
        received = 0
        queue = Queue(address.queue)
        with self.adapter.Reader(queue, url=self.url) as r:
            while received < N:
                m = r.get(1)
                if m is None:
                    break
                m.ack()
                print('#{}] - received: {}]'.format(received, m))
                received += 1
        assert received == N
        print('end')

    def test_not_found(self):
        print('test not-found')
        address = Address(str(uuid4()))
        # sender
        try:
            with self.adapter.Sender(self.url) as sender:
                sender.open()
                sender.send(str(address), 'hello')
                raise Exception('NotFound not raised.')
        except NotFound:
            pass
        # reader
        queue = self.adapter.Queue(address.queue)
        try:
            with self.adapter.Reader(queue, self.url) as reader:
                reader.get()
                raise Exception('NotFound not raised.')
        except NotFound:
            pass

    def test_no_exchange(self):
        print('test builtin (direct) exchange')
        address = Address('test.1')
        queue = self.adapter.Queue(address.queue)
        queue.durable = False
        queue.declare(self.url)
        self.producer_reader(address)

    def test_direct_exchange(self):
        print('test explicit (direct) exchange')
        address = Address('amq.direct/test.2')
        exchange = self.adapter.Exchange(address.exchange, 'direct')
        queue = self.adapter.Queue(address.queue)
        queue.durable = False
        queue.declare(self.url)
        exchange.bind(queue, self.url)
        self.producer_reader(address)

    def test_custom_direct_exchange(self):
        print('test custom (direct) exchange')
        address = Address('test_3.direct/test.3')
        exchange = self.adapter.Exchange(address.exchange, 'direct')
        exchange.durable = False
        exchange.auto_delete = True
        exchange.declare(self.url)
        queue = self.adapter.Queue(address.queue)
        queue.durable = False
        queue.declare(self.url)
        exchange.bind(queue, self.url)
        self.producer_reader(address)

    def test_custom_topic_exchange(self):
        print('test custom (topic) exchange')
        address = Address('test_4.topic/test.4')
        exchange = self.adapter.Exchange(address.exchange, 'topic')
        exchange.durable = False
        exchange.auto_delete = True
        exchange.declare(self.url)
        queue = self.adapter.Queue(address.queue)
        queue.durable = False
        queue.declare(self.url)
        exchange.bind(queue, self.url)
        self.producer_reader(address)

    def test_crud(self):
        print('test CRUD')
        queue = self.adapter.Queue('test.5')
        queue.durable = False
        queue.declare(self.url)
        exchange = self.adapter.Exchange('test_crud_5.direct', 'direct')
        exchange.declare(self.url)
        exchange.bind(queue, self.url)
        queue.delete(self.url)
        exchange.delete(self.url)

    def __call__(self):
        self.test_crud()
        self.test_no_exchange()
        self.test_direct_exchange()
        self.test_custom_direct_exchange()
        self.test_custom_topic_exchange()
        print('DONE')


from setuptools import setup
import os
import sys

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

extra = {}
requirements = ['distribute', 'docutils', 'requests'],
tests_require = ['nose', 'coverage', 'Mock']

# In case we use python3
if sys.version_info >= (3,):
    extra['use_2to3'] = True

if sys.version_info <= (2, 6):
    requirements.append('simplejson')

setup(
    name="docar",
    version="0.5.5",
    packages=['docar', 'docar.backends'],
    include_package_data=True,
    #zip_safe=False,  # Don't create egg files, Django cannot find templates
                     # in egg files.
    install_requires=requirements,
    tests_require=tests_require,
    setup_requires='nose',
    test_suite="nose.collector",
    extras_require={'test': tests_require},

    author="Christo Buschek",
    author_email="crito@30loops.net",
    url="https://github.com/30loops/python-docar",
    description="Create resource oriented architectures for RESTful client \
and server applications.",
    long_description=read('README.rst'),
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Web Environment',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Internet :: WWW/HTTP',
    ],
    **extra
)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from bincrafters import build_template_default
from conans import tools
import shutil
import os

package_name = "corrade"
package_version = "2019.01"
package_download_sha256sum = "3396a081c46f3fd1eb84c42bf626191439aa1fd4e69090663cdd0d46355572c4"
package_download_url = "https://github.com/Croydon/corrade/archive/" + package_version + ".zip"
extracted_dir = package_name + "-" + package_version

# This conan file is getting maintained upstream at https://github.com/mosra/corrade
# while we are providing CI / CD
# download the release and move it accordingly to the places our scripts are expecting the files

tools.get(package_download_url, sha256=package_download_sha256sum)

files_list = os.listdir(extracted_dir)
for files in files_list:
    if not os.path.exists("./" + files):
        shutil.move(os.path.join(extracted_dir, files), "./")
    else:
        shutil.copy(os.path.join(extracted_dir, files), "./")

tools.rmdir(extracted_dir)

if __name__ == "__main__":

    builder = build_template_default.get_builder()

    builder.run()


import json
import requests
from Config import exchanger
class APIException(Exception):
    pass
class Convertor:
    @staticmethod
    def get_price(values):
        if len(values) != 3:
            raise APIException("Неверное количество параметров")
        quote, base, amount = values

        if quote == base:
            raise APIException(f'Невозможно перевести одинаковые валюты {base}')

        try:
            quote_formatted = exchanger[quote]
        except KeyError:
            raise APIException(f'Не удалось обработать валюту {quote}')

        try:
            base_formatted = exchanger[base]
        except KeyError:
            raise APIException(f'Не удалось обработать валюту {base}')

        try:
            amount = float(amount)
        except ValueError:
            raise APIException(f'Не удалось обработать количество {amount}')

        r = requests.get(f'https://min-api.cryptocompare.com/data/price?fsym={quote_formatted}&tsyms={base_formatted}')
        print(r)

        result = float(json.loads(r.content)[base_formatted]) * amount
        return round(result, 3)


import matplotlib.pyplot as plt
from utils import *

def metrics(opt):
    path = opt.ckpt_path
    accs = []
    sens = []
    spes = []
    aucs = []
    for exp in range(opt.num_exp):
        n_fold = opt.num_fold
        matrix = [[0, 0], [0, 0]]
        preds = []
        labels = []
        for i in range(n_fold):

            with open(path + '/exp{}/fold{}_score_test.txt'.format(exp, i), 'r') as f:
                file = f.readlines()
                for j in range(len(file)):
                    data = file[j].split("__")
                    tmp = [data[0], data[1]]
                    tmp = np.array(tmp).astype(np.float32)
                    preds.append(tmp)
                    label = data[2].split()[0]
                    labels.append(round(float(label)))
            f.close()
        matrix = matrix_sum(matrix, get_confusion_matrix(preds, labels))
        acc = (matrix[0][0] + matrix[1][1]) / (matrix[0][0] + matrix[0][1] + matrix[1][0] + matrix[1][1])
        sen = matrix[1][1] / (matrix[0][1] + matrix[1][1])
        spe = matrix[0][0] / (matrix[0][0] + matrix[1][0])
        auc = get_auc(preds, labels)
        print(matrix)

        print("acc: {:.4}%".format(acc*100))

        print("sen: {:.4}%".format(sen*100))

        print("spe: {:.4}%".format(spe*100))

        print("auc: {:.4}".format(auc))

        plt.title('Results of HC vs. MCI classification')

        plt.bar(0, acc, error_kw={'ecolor': '0.2', 'capsize': 6}, alpha=0.7, label='ACC')
        plt.bar(1.5, sen, error_kw={'ecolor': '0.2', 'capsize': 6}, alpha=0.7, label='SEN')
        plt.bar(3, spe, error_kw={'ecolor': '0.2', 'capsize': 6}, alpha=0.7, label='SPE')

        plt.xticks([0, 1.5, 3], ['ACC', 'SEN', 'SPE'])
        plt.legend(loc=2, bbox_to_anchor=(1.05, 1.0), borderaxespad=0.)

        plt.savefig(path + "/exp{}/res.png".format(exp))

        accs.append(acc)
        sens.append(sen)
        spes.append(spe)
        aucs.append(auc)

    print("=> Mean of test acc : {:.2f} and Std of test acc : {:.2f}".format(np.mean(accs) * 100, np.std(accs) * 100))
    print("=> Mean of test sen : {:.2f} and Std of test sen : {:.2f}".format(np.mean(sens) * 100, np.std(sens) * 100))
    print("=> Mean of test spe : {:.2f} and Std of test spe : {:.2f}".format(np.mean(spes) * 100, np.std(spes) * 100))
    print("=> Mean of test auc : {:.4f} and Std of test auc : {:.4f}".format(np.mean(aucs), np.std(aucs)))


from regexast import *
import pydot
import copy
import pdb

class FSM(object):
    def __init__(self, letter_cls):
        self.letter_cls = letter_cls
        self.initial_state = None
        self.states = []
        self.state_count = 1

    def match(self, letter_list):
        state = self.initial_state
        for letter in letter_list:
            state = state.transition(letter)
            if state is None:
                return False

        return state.terminal

    def get_state(self, id):
        for state in self.states:
            if state._id == id:
                return state
        else:
            return None

    def clear_visited(self):
        for state in self.states:
            state.clear_visited()

    def add_state(self, state):
        state._id = self.state_count
        self.state_count += 1
        self.states += [state]

    def draw(self, filename, show_dead=False):
        graph = pydot.Dot(graph_type='digraph')
        state_hash = {}
        def _draw(state):
            if not state.visited:
                node = pydot.Node(str(state._id), label=('"%s"' % str(state._id)))
                if state.terminal:
                    node.set_shape('"doublecircle"')
                else:
                    node.set_shape('"circle"')

                graph.add_node(node)
                state_hash[state] = node
                state.visit()

                for transition_set, new_state in state.edges:
                    if new_state is not self.dead_state or show_dead:
                        new_node = _draw(new_state)
                        letter_str_list = []
                        for letter_id in list(transition_set):
                            if letter_id == -1:
                                letter_str_list += ["0"]
                            else:
                                letter_str_list += [str(self.letter_cls(letter_id))]

                        edge_label = '"' +  ", ".join(letter_str_list) + '"'
                        graph_edge = pydot.Edge(node, new_node)
                        graph_edge.set_label(edge_label)
                        graph.add_edge(graph_edge)

                return node
            else:
                return state_hash[state]

        _draw(self.initial_state)
        graph.write_png(filename)
        self.clear_visited()

class NFSM(FSM):
    def __init__(self, letter_cls):
        super(NFSM, self).__init__(letter_cls)
        self.make_dead_state()

    def make_dead_state(self):
        self.dead_state = FSMState([])
        self.dead_state._id = 0
        self.states += [self.dead_state]

    def reverse_ids(self):
        def _reverse_ids(state):
            if not state.visited:
                state._id = self.state_count - state._id
                state.visit()
                for _, next_state in state.edges:
                    _reverse_ids(next_state)

        _reverse_ids(self.initial_state)
        self.clear_visited()

    def _connect_dead_state(self, state):
        dead_transitions = set()
        for letter in self.letter_cls.all():
            if letter.id() not in state.valid_transitions():
                dead_transitions = dead_transitions | {letter.id()}

        state.add_edges([(dead_transitions, self.dead_state)])

    def connect_dead_state(self):
        for state in self.states:
            if state is not self.dead_state:
                self._connect_dead_state(state)

    def is_impossible(self, state):
        if state is self.dead_state:
            return False

        if state.terminal:
            return False

        for letters, subs_state in state.edges:
            if subs_state is not self.dead_state and \
                    not (len(letters) == 1 and tuple(letters)[0] == -1):
                return False

        return True

    def remove_impossible(self, state):
        # pdb.set_trace()
        epsilon_edges = state.get_epsilon_edges()
        for other_state in self.states:
            if state is other_state:
                continue

            edges_to = other_state.get_edges_to(state)
            if len(edges_to) != 0:
                other_state.remove_edges_to(state)
                new_edges = []
                for incoming_letters, _ in edges_to:
                    for _, destination_state in epsilon_edges:
                        new_edges += [(incoming_letters, destination_state)]

                other_state.add_edges(new_edges)

    def remove_impossible_states(self):
        for state in self.states:
            if self.is_impossible(state):
                self.remove_impossible(state)

    def from_regex(self, ast):
        self.initial_transitions = self._from_regex(ast)
        self.initial_state = FSMState([self.initial_transitions])
        self.add_state(self.initial_state)
        self.connect_dead_state()
        self.reverse_ids()
        self.clear_visited()
        self.remove_impossible_states()

    def _from_regex(self, ast, next_transition=None):
        if ast.type == RegexASTNode.CAT_EXPR:
            transition_B = self._from_regex(ast.right, next_transition=next_transition)
            mid_state = FSMState([transition_B])
            self.add_state(mid_state)
            transition_A = self._from_regex(ast.left, next_transition=(-1, mid_state))
            return transition_A
        elif ast.type == RegexASTNode.LETTER:
            state_B = None
            if next_transition is None:
                state_B = FSMState([], terminal=True)
                self.add_state(state_B)
            else:
                _, state_B = next_transition

            return (ast.value.id(), state_B)
        elif ast.type == RegexASTNode.EPSILON:
            state_B = None
            if next_transition is None:
                state_B = FSMState([], terminal=True)
                self.add_state(state_B)
            else:
                _, state_B = next_transition

            return (-1, state_B)
        elif ast.type == RegexASTNode.OR_EXPR:
            if next_transition is None:
                final_state = FSMState([], terminal=True)
                self.add_state(final_state)
                next_transition = (-1, final_state)

            transitions_A = self._from_regex(ast.left, next_transition=next_transition)
            transitions_B = self._from_regex(ast.right, next_transition=next_transition)
            or_initial_state = FSMState([transitions_A, transitions_B])
            self.add_state(or_initial_state)
            return (-1, or_initial_state)
        elif ast.type == RegexASTNode.REP_EXPR:
            terminal = (next_transition is None)
            loop_state = FSMState([], terminal=terminal)
            loop_init_transition = self._from_regex(ast.left, next_transition=(-1, loop_state))

            if next_transition is not None:
                loop_state.add_edges([loop_init_transition, next_transition])
            else:
                loop_state.add_edges([loop_init_transition])

            self.add_state(loop_state)
            return (-1, loop_state)
        else:
            raise Exception("Unknown AST type.")

class FSMState(object):
    def __init__(self, edges, id=None, terminal=False):
        # edges is expected to be of the form
        #    [ ({letter1_id1, letter1_id2}, new_state1), ... , (lettern_id, new_staten)]
        self.edges = []
        self._id = id
        self.add_edges(edges)
        self.visited = False

        self.terminal = terminal

    def __hash__(self):
        return hash(self._id)

    def clear_visited(self):
        self.visited = False

    def visit(self):
        self.visited = True

    def get_epsilon_edges(self):
        epsilon_edges = []
        for letters, next_state in self.edges:
            if len(letters) == 1 and tuple(letters)[0] == -1:
                epsilon_edges += [(letters, next_state)]

        return epsilon_edges

    def get_edges_to(self, state):
        edges_to = []
        for letters, next_state in self.edges:
            if next_state is state:
                edges_to += [(letters, next_state)]

        return edges_to

    def remove_edges_to(self, state):
        self.edges = [edge for edge in self.edges if not edge[1] is state]

    def add_edges(self, edges):
        for edge, new_state in edges:
            if type(edge) is set:
                self.edges += [(edge, new_state)]
            else:
                self.edges += [({edge}, new_state)]

    # only call on deterministic nodes
    def transition(self, letter):
        for edge_letter_set, state in self.edges:
            if letter.id() in edge_letter_set:
                return state

        return None

    def valid_transitions(self):
        letters = set()
        for transition_letters, next_state in self.edges:
            letters = letters | transition_letters

        return letters

    def nd_transition(self, letter):
        new_states = set()
        if not self.visited:
            self.visit()
            for edge_letter_set, state in self.edges:
                if letter in edge_letter_set:
                    new_states = new_states | {state}

                if -1 in edge_letter_set:
                    new_states = new_states | state.nd_transition(letter)

        return new_states

    def __eq__(self, other):
        return self._id == other._id

class DFSM(FSM):
    def from_regex(self, ast):
        nfsm = NFSM(self.letter_cls)
        nfsm.from_regex(ast)
        self.from_nfsm(nfsm)

    def from_nfsm(self, nfsm):
        self.dead_state = FSMSuperState({nfsm.dead_state}, [])
        self.add_state(self.dead_state)
        self.initial_state = FSMSuperState({nfsm.initial_state}, [])
        self.add_state(self.initial_state)
        self._from_nfsm(nfsm, self.initial_state)
        self.clear_visited()

    def get_by_ref_states(self, ref_states):
        for state in self.states:
            if ref_states == state.component_states:
                return state
        else:
            return None

    def make_state(self, component_states):
        ref_state = self.get_by_ref_states(component_states)
        if ref_state is None:
            ref_state = FSMSuperState(component_states, [])
            self.add_state(ref_state)

        return ref_state

    def _from_nfsm(self, nfsm, state):
        if not state.visited:
            state.visit()
            paths = []

            for letter in self.letter_cls.all():
                this_letter_states = state.nd_transition(letter.id(), nfsm)
                next_state = self.make_state(this_letter_states)

                paths += [(letter.id(), next_state)]

            state.add_edges(paths)
            for _, next_state in state.edges:
                self._from_nfsm(nfsm, next_state)

class FSMSuperState(FSMState):
    # component_states expected to be a set
    def __init__(self, component_states, edges, id=None, terminal=False):
        super(FSMSuperState, self).__init__(edges, id=id, terminal=terminal)
        self.component_states = component_states
        self.set_terminal()

    def set_terminal(self):
        for component_state in self.component_states:
            self.terminal |= component_state.terminal

    def __eq__(self, other):
        if type(other) is FSMState:
            if len(self.component_states) == 1:
                return self.component_states[0] == other

        if type(other) is not FSMSuperState:
            return False

        same = True
        for component_state in self.component_states:
            same = same and (component_state in other.component_states)

    def nd_transition(self, letter_id, nfsm):
        new_states = set()
        for component_state in self.component_states:
            new_states = new_states | component_state.nd_transition(letter_id)
            nfsm.clear_visited()

        return new_states


import os
import json
from argparse import Namespace

import onmt
from onmt.utils.logging import init_logger
from onmt.utils.misc import split_corpus
from onmt.translate.translator import build_translator
from onmt.translate.translator import Translator
import onmt.opts as opts
from onmt.utils.parse import ArgumentParser

from services import Service, MissingLanguage

def _load_model_description(model_dir):
    '''
    Reads configuration parameters and training metrics from JSON files stored with the AllenNLP model.
    Returns a dictionary that can be used to populate the 'models' field of the service description.
    '''
    desc = {}
    try:
        descr_fn = os.path.join(model_dir, 'model_params.json')
        if os.path.isfile(descr_fn):
            with open(descr_fn) as descr_file:
                config = json.load(descr_file)
                desc['datasets'] = config.get('datasets')
                desc['params'] = config.get('params')
                desc['training-time'] = config.get('training_time')
                desc['metrics'] = config.get('metrics')
    except (IOError, ValueError):
        pass
    return desc

class Summarization(Service):
    def __init__(self, models_dir='models/opennmt/summarization'):
        Service.__init__(self, 'abstractive-summarization', 'opennmt', ['parse'])
        # define opt values for the summarisation task
        self.models = {}
        self.descriptions = {}
        for lang in os.listdir(models_dir):
            if len(lang) == 2:
                self.models[lang] = self._load_model(os.path.join(models_dir, lang))
                self.descriptions[lang] = _load_model_description(os.path.join(models_dir, lang))

    @staticmethod
    def _load_model(basedir):
        with open(os.path.join(basedir,'config_dict.json'), 'r') as fp:
            baseopt = json.load(fp)

        opt = Namespace(**baseopt)
        opt.models = [os.path.join(basedir, 'cnndm_step_200000.pt')]
        opt.gpu = -1
        translator = build_translator(opt, report_score=True)

        return translator

    def _summarize_text(self, model, text, debug):
        score, pred = model.translate(src=[text], batch_size=20)
        summary = (pred[0])[0].replace(' .', '.').replace('<t> ', '').replace(' </t>', '').strip()
        return {
            'text': summary,
            'score': (score[0][0]).item()
        }

    def run(self, request, response):
        lang = request['lang']
        if lang in self.models:
            model = self.models[lang]
            text = request['text']
            parsing = response['parse']
            debug = request.get('debug', False)

            sentences = []
            for sentence in parsing:
                tokens = []
                for item in sentence:
                    if 'text' in item:
                        token = item['text'].strip()
                    else:
                        token = text[item['start']:item['end']].strip()
                    if token != '':
                        tokens.append(token)
                sentences.append('<t> ' + ' '.join(tokens) + ' </t>')
            summary_response = self._summarize_text(model,' '.join(sentences),debug)

            item = {
                'summary': summary_response['text']
            }

            if debug:
                item['summarization_ratio'] = len(summary_response['text']) / len(text)
                item['prediction_score'] = summary_response['score']

            return item
        else:
            raise MissingLanguage(lang)

    def describe(self):
        result = super().describe()
        result['langs'] = list(self.descriptions.keys())
        result['models'] = self.descriptions
        return result

class Translation(Service):
    def __init__(self, models_dir='models/opennmt/translation'):
        Service.__init__(self, 'translation', 'opennmt', ['parse'])
        # define opt values for the summarisation task
        self.models = {}
        self.descriptions = {}
        for lang in os.listdir(models_dir):
            if len(lang) == 5:
                self.models[lang] = self._load_model(os.path.join(models_dir, lang),lang)
                self.descriptions[lang] = _load_model_description(os.path.join(models_dir, lang))

    @staticmethod
    def _load_model(basedir,lang):
        with open(os.path.join(basedir,'config_dict.json'), 'r') as fp:
            baseopt = json.load(fp)

        opt = Namespace(**baseopt)
        opt.models = [os.path.join(basedir, lang + '_step_200000.pt')]
        opt.gpu = -1
        translator = build_translator(opt, report_score=True)

        return translator

    def _translate_text(self, model, text, debug):
        score, pred = model.translate(src=[text], batch_size=20)
        summary = (pred[0])[0].replace(' .', '.').replace('<t> ', '').replace(' </t>', '').strip()
        return {
            'text': summary,
            'score': (score[0][0]).item()
        }

    def run(self, request, response):
        lang_pair = request['lang'] + '-' + request['target-lang']
        trl_sentences = []
        scores = []
        sentencewise = True

        if lang_pair in self.models:
            model = self.models[lang_pair]
            text = request['text']
            parsing = response['parse']
            debug = request.get('debug', False)
            sentences = []
            for sentence in parsing:
                tokens = []
                for item in sentence:
                    if 'text' in item:
                        token = item['text'].strip()
                    else:
                        token = text[item['start']:item['end']].strip()
                    if token != '':
                        tokens.append(token)
                if sentencewise:
                    trl_output = self._translate_text(model,' '.join(tokens),debug)
                    trl_sentences.append(trl_output['text'])
                    if debug:
                        scores.append(trl_output['score'])
                else:
                    sentences.append(' '.join(tokens))

            # Here translate sentence by sentence (otherwise it seems that it will just translate + summarize)
            item = {}

            if sentencewise:
                item['translation'] = '  '.join(trl_sentences)
                if debug:
                    #item['summarization_ratio'] = len(summary_response['text'])/len(text)
                    item['prediction_score'] = scores
            else:
                trl_output = self._translate_text(model, ' '.join(sentences), debug)
                item['translation'] = trl_output['text']
                item['prediction_score'] = trl_output['score']

            return item
        else:
            raise MissingLanguage(lang_pair)

    def describe(self):
        result = super().describe()
        result['langs'] = list(x[:2] for x in self.descriptions.keys())
        result['models'] = self.descriptions
        result['extra-params'] = [{
            'name': 'target-lang',
            'type': 'string',
            'required': True
        }]
        return result


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 16 12:00:18 2019

@author: parkerglenn
"""
import os
import pandas as pd
import numpy as np
import sklearn.manifold
import matplotlib.pyplot as plt
import nltk
import regex as re
import re
import codecs
import csv
import glob
import multiprocessing
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
from nltk.tokenize import word_tokenize, sent_tokenize
stemmer = SnowballStemmer("english")
import math
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import seaborn as sns
import sklearn

os.chdir("/Users/parkerglenn/Desktop/DataScience/Article_Clustering")
df = pd.read_csv("/Users/parkerglenn/Desktop/DataScience/Article_Clustering/csv/all_GOOD_articles.csv")
labels_df= pd.read_csv("/Users/parkerglenn/Desktop/DataScience/Article_Clustering/Google_Drive/Article_Classification122.csv")
#Deletes unnecessary columns
df = df.drop(df.columns[:12], axis = 1)
#Sets manageable range for working data set
new_df = df[5000:6000]
#Gets info in list form to be later called in kmeans part

corpus = []
for text in new_df['content']:
    corpus.append(text)

titles = []
for title in new_df["title"]:
    titles.append(str(title))
#labels_df starts at df[5000] so we're good on the matching of labels to content
events = []
for event in labels_df["Event"][:1000]:
    events.append(str(event))

import spacy
from spacy import gold
from spacy.gold import iob_to_biluo
nlp = spacy.load('en_core_web_md', disable=['parser','tagger','textcat'])
nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
english_stopwords = stopwords.words('english')
from sklearn.feature_extraction import stop_words

##############################################################################
###################HYPER-PARAMETER TESTING####################################
##############################################################################
from hyper_parameter_functions import tokenize_and_stem_NER, do_tfidf, HAC, success

hyper_params = pd.DataFrame(columns = ["ents_rate", "person_rate","f_score","s_score","samples used"])

for person_rate in np.linspace(1,7,20):
    for ents_rate in np.linspace(1,7,20):
        tfidf_matrix = do_tfidf(ents_rate, person_rate)
        HAC(tfidf_matrix)
        cols = [pd.Series([ents_rate,person_rate,f_score,s_score,num2],index=hyper_params.columns)]
        hyper_params = hyper_params.append(cols)

hyper_params.plot.scatter(x="f_score", y = "s_score")

f_scores = [x for x in hyper_params["f_score"]]
s_scores = [x for x in hyper_params["s_score"]]
person_rate = [x for x in hyper_params["person_rate"]]
ents_rate = [x for x in hyper_params["ents_rate"]]

cmap = pyplot.cm.cubehelix
dimensions = (20,20)
fig, ax = pyplot.subplots(figsize=dimensions)
sns.heatmap(dist, vmin = 0, vmax = 1, cmap = cmap).set_title("Tfidf Distances Between Articles", fontsize = 15)

import seaborn as sns

"""F1 Score distribution across hyperparameters
Best F1 Score: ents_rate = 1.63, person_rate = 2.57, f_score = .927 on 412 articles. BUT s_score is a measly .0788.
"""
f_data = pd.DataFrame({"Person Rate": [round(x,2) for x in person_rate], "Entity Rate":[round(x,2) for x in ents_rate], "Z": hyper_params.f_score})
f_data_pivoted = f_data.pivot("Person Rate","Entity Rate","Z")
ax1 = sns.heatmap(f_data_pivoted, cmap = plt.cm.hot)
cbar = ax1.collections[0].colorbar
cbar.set_label('F1 Score', labelpad=15)
ax1.invert_yaxis()
plt.show()

"""Silhouette Score distribution across hyperparameters

More direct correlation here than in F1 Score. Most notably, as the rates increase, so does S Score. This is due to
the fact that the rating of words in articles becomes more radical; articles that are different from each other, i.e. share no
entities, are now much more distant than before. Even more specifically, when the Entity Rate and Person Rate are dissimilar, the
S Score is the highest. Again, the same radical rating phenomena: By limiting the amount of weighted features, those that are weighted
make the article more of an outlier than before."""
s_data = pd.DataFrame({"Person Rate": [round(x,2) for x in person_rate], "Entity Rate":[round(x,2) for x in ents_rate], "Z": hyper_params.s_score})
s_data_pivoted = s_data.pivot("Person Rate","Entity Rate","Z")
ax2 = sns.heatmap(s_data_pivoted, cmap = plt.cm.hot)
cbar = ax2.collections[0].colorbar
cbar.set_label('Silhouette Score', labelpad=15)
ax2.invert_yaxis()
plt.show()

"""Cumulative Total
Highest: ents_rate = 6.368, person_rate = 2.263, f_score = 0.922, s_score = 0.093

"""
sf_data = pd.DataFrame({"Person Rate": [round(x,2) for x in person_rate], "Entity Rate":[round(x,2) for x in ents_rate], "Z": hyper_params.s_score + hyper_params.f_score})
sf_data_pivoted = sf_data.pivot("Person Rate","Entity Rate","Z")
ax3 = sns.heatmap(sf_data_pivoted, cmap = plt.cm.hot)
cbar = ax3.collections[0].colorbar
cbar.set_label('Composite Score', labelpad=15)
ax3.invert_yaxis()
plt.show()

"""Little thing to find best cumulative score and rates that achieved it."""
hightot = 0
for tup in enumerate(hyper_params["f_score"]):
    tot = tup[1] + hyper_params.loc[tup[0]]["s_score"]
    if hightot < tot:
        hightot = tot
        best_scores = (hyper_params.loc[tup[0]]["ents_rate"],hyper_params.loc[tup[0]]["person_rate"])
best_scores


from typing import List
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.setting import Setting, SettingCreate

from sqlmodel.sql.expression import Select, SelectOfScalar

SelectOfScalar.inherit_cache = True  # type: ignore
Select.inherit_cache = True  # type: ignore

class CRUDSetting:
    def __init__(self) -> None:
        pass

    async def get(self, id: int, session: AsyncSession) -> Setting:
        return await session.get(Setting, id)

    async def get_all(self, session: AsyncSession) -> List[Setting]:
        query = select(Setting)
        result = await session.execute(query)
        return result.scalars().all()

    async def add(self, data: SettingCreate, session: AsyncSession) -> Setting:
        obj = Setting.from_orm(data)
        session.add(obj)
        await session.commit()
        await session.refresh(obj)
        return obj

    async def delete(self, data: Setting, session: AsyncSession) -> Setting:
        delete = await session.get(Setting, data.id)
        obj = await session.delete(delete)
        await session.commit()
        return obj

    async def update_value(self, id: int, value: float, session: AsyncSession):
        obj = await session.get(Setting, id)
        obj.value = float(value)
        session.add(obj)
        await session.commit()
        await session.refresh(obj)
        return obj

    async def exists(self, id: int, session: AsyncSession) -> bool:
        return await session.get(Setting, id) is not None


from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import reactor
from twisted.internet.task import deferLater
from twisted.python.log import err

# first consume 5, second consume 5 s
class DelayedResource(Resource):

    def _delayRender(self, request):
        request.write("Sorry to keep you waiting.")
        request.finish()

    def _responseFailed(self, failure, call):
        call.cancel()
        err(failure, "Async response demo interrupted response")

    def render_GET(self, request):
        call = reactor.callLater(5, self._delayRender, request)
        request.notifyFinish().addErrback(self._responseFailed, call)
        return NOT_DONE_YET

root = Resource()
root.putChild("x", DelayedResource())
factory = Site(root)
reactor.listenTCP(8888, factory)
reactor.run()


import json
import os.path
from werkzeug.utils import secure_filename
from flask import render_template, request, redirect, url_for, flash, abort, session, jsonify, Blueprint

bp = Blueprint("url_short", __name__)

@bp.route("/")
def home():
    return render_template("home.html", codes=session.keys())

@bp.route("/your-url", methods=["POST", "GET"])
def about():
    if request.method == "POST":
        if os.path.exists("../urls.json"):
            with open("../urls.json", "r") as urls_file:
                urls = json.load(urls_file)
        if request.form["code"] in urls.keys():
            flash("That short name has already been taken. Please, select another name")
            return redirect(url_for("url_short.home"))
        if "url" in request.form.keys():
            urls = {request.form["code"]: {"url": request.form["url"]}}
        else:
            file = request.files["file"]
            full_name = request.form["code"] + secure_filename(file.filename)
            file.save("/home/vlad/Desktop/Learning/flask_essential_training/url_short./static/user_files/" + full_name)
            urls = {request.form["code"]: {"file": full_name}}
        with open("../urls.json", "w") as urls_file:
            json.dump(urls, urls_file)
            session[request.form["code"]] = True
        return render_template("your_url.html", code=request.form["code"])
    else:
        return redirect(url_for("url_short.home"))

@bp.route("/<string:code>")
def redirect_to_url(code):
    if os.path.exists("../urls.json"):
        with open("../urls.json") as urls_file:
            urls = json.load(urls_file)
            if code in urls.keys():
                if "url" in urls[code].keys():
                    return redirect(urls[code]["url"])
                else:
                    return redirect(url_for("url_short.static", file="user_files/" + urls[code]["file"]))
    return abort(404)

@bp.errorhandler(404)
def page_not_found(error):
    return render_template("page_not_found.html"), 404

@bp.route("/api")
def session_api():
    return jsonify([session.keys()])


from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait

from Pages.BasePage import BasePage
from utility.ReusableMethods import Generic

class CheckoutPage(Generic):
    text = None

    def __init__(self, driver):
        super().__init__(driver)

    CHECKOUT_BUTTON = (By.CSS_SELECTOR, "button[class='btn btn-success']")
    PURCHASE_BUTTON = (By.CSS_SELECTOR, "input[value='Purchase']")
    PURCHASE_TEXT = (By.ID, "country")
    COUNTRY_LIST = (By.XPATH, "//div[@class='suggestions']//li//a")
    SUCCESS_TEXT = (By.XPATH, " //div[contains(@class,'alert alert-success alert-dismissible')]")

    def click_checkout_button_at_checkout_page(self):
        return self.driver.find_element(*CheckoutPage.CHECKOUT_BUTTON)

    def enter_Purchase_field(self):
        return self.driver.find_element(*CheckoutPage.PURCHASE_TEXT)

    def enter_get_Country(self, text, value):
        self.enter_text(self.PURCHASE_TEXT, text)
        self.wait_for_element_display(self.COUNTRY_LIST)

        countries = self.driver.find_elements(*CheckoutPage.COUNTRY_LIST)
        for country in countries:
            text = country.text
            # print("the country are:--->"+text)

            if text == value:
                country.click()
                print("the selected country is --->" + text)
                return text
            else:
                return None

    def get_value(self):
        return self.driver.find_element(*CheckoutPage.PURCHASE_TEXT).get_attribute("value")

    def verify_Success_message(self):
        self.click(self.PURCHASE_BUTTON)
        value = self.get_text(self.SUCCESS_TEXT)
        s = value.split()
        return s[1]


import numpy as np
import pandas as pd
import pydotplus

from sklearn import preprocessing
from sklearn.tree import DecisionTreeClassifier
from sklearn import tree
from IPython.display import Image

def show(clf, features, y_types):
    """ 决策树的可视化 """
    dot_data = tree.export_graphviz(clf, out_file=None,
                                    feature_names=features,
                                    class_names=y_types,
                                    filled=True,
                                    rounded=True,
                                    special_characters=True)
    graph = pydotplus.graph_from_dot_data(dot_data)
    graph.write_png(r'DT_Show.png')

# 数据
features = ["age", "work", "house", "credit"]
X_train = pd.DataFrame([
                  ["青年", "否", "否", "一般"],
                  ["青年", "否", "否", "好"],
                  ["青年", "是", "否", "好"],
                  ["青年", "是", "是", "一般"],
                  ["青年", "否", "否", "一般"],
                  ["中年", "否", "否", "一般"],
                  ["中年", "否", "否", "好"],
                  ["中年", "是", "是", "好"],
                  ["中年", "否", "是", "非常好"],
                  ["中年", "否", "是", "非常好"],
                  ["老年", "否", "是", "非常好"],
                  ["老年", "否", "是", "好"],
                  ["老年", "是", "否", "好"],
                  ["老年", "是", "否", "非常好"],
                  ["老年", "否", "否", "一般"]
                  ])
y_train = pd.DataFrame(["否", "否", "是", "是", "否", "否", "否", "是", "是", "是", "是", "是", "是", "是", "否"])
# 因为数据是类别数据，需要进行处理，转化为数值型
# 处理X数据
Pre_X = preprocessing.LabelEncoder()
Pre_X.fit(np.unique(X_train))
X_train = X_train.apply(Pre_X.transform)
print(X_train)
# 处理y数据
Pre_y = preprocessing.LabelEncoder()
Pre_y.fit(np.unique(y_train))
y_train = y_train.apply(Pre_y.transform)
print(y_train)
# 实例化模型
clf = DecisionTreeClassifier()
# 训练
clf.fit(X_train, y_train)
# 可视化
show(clf, features, [str(k) for k in np.unique(y_train)])
# 测试数据
X_test = pd.DataFrame([["青年", "否", "是", "一般"]])
X_test = X_test.apply(Pre_X.transform)
# 预测
y_predict = clf.predict(X_test)
# 展示结果
X_show = [{features[i]: X_test.values[0][i]} for i in range(len(features))]
print("{0} 被分类为： {1}".format(X_show, Pre_y.inverse_transform(y_predict)))