from flask import Flask, Markup, render_template,redirect, url_for, request,make_response
import pandas as pd
import json
from flask import jsonify
import datetime
import pymysql
from sqlalchemy import create_engine

app = Flask(__name__)

session = create_engine('mysql+pymysql://b1988b353f38a9:eb355ccd@us-cdbr-east-02.cleardb.com:3306/heroku_57e05bf7b956609')

@app.route('/fetchalliteams', methods = ['GET', 'POST'])
def fetchalliteams():
    result = pd.read_sql("select * from amazon_shoe_store",con=session)
    #print(result)
    return render_template('index.html',tables=[result.to_html(classes='result', header="true")])

@app.route('/selecteditem', methods = ['GET', 'POST'])
def selecteditem():
    selectedprodid=request.form['productid']
    result = pd.read_sql("select * from amazon_shoe_store where sell_price = {0}".format(float(selectedprodid)),con=session)
    return render_template('index.html',tables=[result.to_html(classes='result', header="true")])

@app.route('/range', methods = ['GET', 'POST'])
def range():
    selectedprodidmin=request.form['productidmin']
    selectedprodidmax = request.form['productidmax']
    result = pd.read_sql("select * from amazon_shoe_store where sell_price >= {0} and sell_price <= {1}".format(float(selectedprodidmin),float(selectedprodidmax)),con=session)
    return render_template('index.html',tables=[result.to_html(classes='result', header="true")])

def define_discount(x,maximum_discount):
    if x> 0 and x <= 20:
        return 'Up to 20% Discount'
    elif x> 20 and x<=50:
        return 'Minimum 20% to Up to 50% Discount'
    else:
        return 'More than 50% Discount Up to {0}%'.format(maximum_discount)

def reviewer_range(x):
    if x>= 1 and x <= 2:
        return 'Rating from 1.0 to 2.0'
    elif x>= 2 and x <= 3:
        return 'Rating from 2.0 to 3.0'
    elif x>= 3 and x <= 4:
        return 'Rating from 3.0 to 4.0'
    elif x>= 4 and x <= 5:
        return 'Rating from 4.0 to 5.0'

@app.route('/charts')
def charts():
    cleaned_dataset = pd.read_sql("select mrp_price,sell_price from amazon_shoe_store",con=session)
    discount = cleaned_dataset[(cleaned_dataset.mrp_price - cleaned_dataset.sell_price)>0][['mrp_price','sell_price']]
    discount['discount_precentage'] = round(((cleaned_dataset.mrp_price - cleaned_dataset.sell_price)/cleaned_dataset.mrp_price)*100,2)
    mimimum_discount = discount.discount_precentage.min()
    maximum_discount = discount.discount_precentage.max()
    discount['category'] = discount['discount_precentage'].apply(lambda x:define_discount(x,maximum_discount))
    d = discount.category.value_counts()
    discount_chart = pd.DataFrame({'discount_category':d.keys(),'number_of_products':d.values})

    xaxis=discount_chart['discount_category'].values.tolist()
    yaxis=discount_chart['number_of_products'].values.tolist()
    labels = xaxis
    values = yaxis

    colors = [
        "#32CD32", "#FDB45C", "#B22222"]
    bar_labels = labels
    bar_values = values
    max_v = discount_chart.number_of_products.max() + 100
    varaiable1 = str(discount_chart.number_of_products.sum())
    varaiable2 = str(mimimum_discount)+'%'
    varaiable3 = str(maximum_discount)+'%'

    return render_template('charts.html',var1=varaiable1,var2=varaiable2,var3=varaiable3, title='Number of products in each discount category', max=max_v, set=zip(values, labels, colors))

@app.route('/charts1')
def charts1():
    cleaned_dataset = pd.read_sql("select rating from amazon_shoe_store",con=session)
    ratings = cleaned_dataset[['rating']]
    rat = ratings.rating.value_counts()
    rating_chart = pd.DataFrame({'rating':rat.keys(),'number_of_products':rat.values})
    rating_chart = rating_chart[['rating','number_of_products']].sort_values('rating').reset_index(drop='index')

    xaxis=rating_chart['rating'].values.tolist()
    yaxis=rating_chart['number_of_products'].values.tolist()

    labels = xaxis
    values = yaxis
    bar_labels = labels
    bar_values = values
    max_v2 = rating_chart.number_of_products.max()+100

    return render_template('charts1.html', title='Rating wise Number of Products', max=max_v2, labels=bar_labels, values=bar_values)

@app.route('/charts2')
def charts2():
    cleaned_dataset = pd.read_sql("select product,sell_price,rating,reviews from amazon_shoe_store",con=session)
    reviewers = cleaned_dataset[['product','sell_price','rating','reviews']]
    reviewers['rating_range'] = reviewers['rating'].apply(lambda x:reviewer_range(x))
    reviewers = reviewers.groupby('rating_range').agg({'reviews':['min','max'],'sell_price':['min','max']}).reset_index()
    print(reviewers)
    reviewers.columns = ['rating','minimum_reviewers','maximum_reviewers','min_price_product','max_price_product']
    xaxis=reviewers['rating'].values.tolist()
    yaxis=reviewers['minimum_reviewers'].values.tolist()
    #print(yaxis)
    labels = xaxis
    values = yaxis

    bar_labels = labels
    bar_values = values
    max1 = reviewers['minimum_reviewers'].max() +1

    bar_labels1 = xaxis
    bar_values1 = reviewers['maximum_reviewers'].values.tolist()
    max2 = reviewers['maximum_reviewers'].max() +100

    bar_labels2 = xaxis
    bar_values2 = reviewers['min_price_product'].values.tolist()
    max3 = reviewers['min_price_product'].max() +10

    bar_labels3 = xaxis
    bar_values3 = reviewers['max_price_product'].values.tolist()
    max4 = reviewers['max_price_product'].max() + 50
    print(max1,max2,max3,max4)
    return render_template('charts2.html', title='', max1=max1,max2=max2,max3=max3,max4=max4, labels=bar_labels, values=bar_values,labels1=bar_labels1, values1=bar_values1,labels2=bar_labels2, values2=bar_values2,labels3=bar_labels3, values3=bar_values3)

@app.route("/")
def main():
    return render_template('index.html')

if __name__ == '__main__':
    app.run(host='localhost',debug=True)


"""
Solver for the Crank-Nicolson implementation of heat transfer
One dimension
Constant properties
Aluminium block for conductive losses in the back face
Accomodates linear or non-linear surface boundary condition
"""

# import libraries
import numpy as np
import pandas as pd

def general_solver(heat_flux, temp_initial, temp_air, k, alpha, x_grid,t_grid, upsilon,
                   bc_surface, sigma):
    """
    General function to implement CN.
    Returns dictionary that includes all calculated temperatures for given boundary condition

    CN Scheme:
    ---------
        [A]{T^n+1} = [B]{T^n} = [b]

    Parameters:
    ----------
    heat_flux: np.array
        array of IHF in time

    temp_initial: int
        initial temperature in K

    temp_air: int
        air (infinity) temperature for calculating convective losses

    k: int
        thermal conductivity in W/mK

    alpha: float
        thermal diffusivity in m2/s

    x_grid: np.array
        one dimentional spatial array

    t_grid: np.array
        array over which the algorithm advances in time

    upsilon: Fourier number divided by 2
        list

    bc_surface: list
        Type of surface losses to be considered and necessary parameters

    sigma: Stefan Boltzman constant
        float

    Returns:
    -------
    Temperature: pd.DataFrame
        one-dimensional temperature profile in time

    """

    # temperatures are reported as a data frame, where each column is a step in time
    temperatures = pd.DataFrame(columns = [n for n in t_grid])

    # extract the necessary parameters to determine the surface heat losses
    if bc_surface[0] == "linear":
        h = bc_surface[1] + bc_surface[2]
        hc = 0
        emissivity = 0
    elif bc_surface[0] == "non-linear":
        h = 0
        hc = bc_surface[1]
        emissivity = bc_surface[2]

    # initialize temperature arrays for present and future temperatures
    T = np.zeros_like(x_grid) + temp_initial
    Tn = np.zeros_like(x_grid)

    # iterate over each time step
    temperatures.iloc[:,0] = T
    for j, t in enumerate(t_grid[:-1]):

        # create tri-diagonal matrix A
        A = tridiag_matrix(bc_surface_type = bc_surface[0], upsilon = upsilon,
                           space_divisions = len(x_grid), dx = x_grid[1] - x_grid[0],
                           k = k, T = T, h = h, hc = hc, emissivity = emissivity, sigma = sigma)

        # create vector b
        b = vector_b(bc_surface_type = bc_surface[0], upsilon = upsilon,
                     space_divisions = len(x_grid), dx = x_grid[1] - x_grid[0],
                     k = k, T = T, T_air = temp_air, heat_flux = heat_flux, h = h, hc = hc,
                              emissivity = emissivity, sigma = sigma, j = j)

        # calculate value of future temperature
        Tn = np.linalg.solve(A,b)

        # update present temperature
        T = Tn.copy()

        # store temperature profile at this time in the data frame
        temperatures.iloc[:, j+1] = Tn

    return temperatures

# function to create tri-diagonal matrix
def tridiag_matrix(bc_surface_type, upsilon, space_divisions, dx, k, T, h, hc, emissivity, sigma):
    """
    Creates tridiagonal matrix A
    Linear system to be solved is Ax = b, and x represents temperature values at time n+1

    Parameters:
    ----------
    bc_surface_type: str
        boundary condition at the surface

    upsilon: float
        Fourier number divided by 2. Upsilon = alpha*dt/2*dx2

    space_divisions: int
        number of nodes in the spatial domain

    dx: float
        size of cell in space domain in m

    k: float
        thermal conductivity in W/mK

    T: np.array
        array of present temperatures

    h: int
        total heat transfer coefficient for the linearised surface boundary condition

    hc: int
        convective heat transfer coefficient

    emissivity: float
        surface emmisivity, assumed constant

    sigma: float
        Stefan Boltzman constant

    Return:
    ------

    A: matrix to be inverted
        np.array

    """
    # create tri-diagonal matrix
    A = np.diagflat([-upsilon for i in range(space_divisions - 1)], -1) +\
        np.diagflat([1 + 2 * upsilon for i in range(space_divisions)]) +\
        np.diagflat([-upsilon for i in range(space_divisions - 1)], 1)

    # adjust matrix depending on the boundary condition at the exposed surface
    if bc_surface_type == "linear":
        A[0,0] = 1 + 2*upsilon + 2*upsilon*dx*h/k
        A[0,1] = -2*upsilon

    elif bc_surface_type == "non-linear":
        A[0,0] = 1 + 2*upsilon + 2*dx*hc*upsilon/k+ 8*emissivity*sigma*dx*upsilon*T[0]**3/k
        A[0,1] = -2*upsilon

    # adjust matrix for the back boundary conditions
    A[-1, -2] = - 2 * upsilon
    A[-1, -1] = 1 + 2 * upsilon

    return A

def vector_b(bc_surface_type, upsilon, space_divisions, dx, k, T, T_air, heat_flux, h, hc, emissivity, sigma, j):
    """
    Calculates vector b. Right hand side of linear system of equations

    Parameters:
    ----------
    bc_surface_type: str
        boundary condition at the surface

    upsilon: float
        Fourier number divided by 2. Upsilon = alpha*dt/2*dx2

    space_divisions: int
        number of nodes in the spatial domain

    dx: float
        size of cell in space domain in m

    k: float
        thermal conductivity in W/mK

    T: np.array
        array of present temperatures

    T_air: int
        air (infinity) temperature for calculating convective losses

    heat_flux: np.array
        full array of IHF vs time

    h: int
        total heat transfer coefficient for the linearised surface boundary condition

    hc: int
        convective heat transfer coefficient

    emissivity: float
        surface emmisivity, assumed constant

    sigma: float
        Stefan Boltzman constant

    j: int
        present iteration number

    Returns:
    -------
    b: np.array
        vector to solve linear system of equations
    """

    # matrix B, similar to matrix A but multiplies T at present
    B = np.diagflat([upsilon for i in range(space_divisions - 1)], -1) +\
        np.diagflat([1 - 2 * upsilon for i in range(space_divisions)]) +\
        np.diagflat([upsilon for i in range(space_divisions - 1)], 1)

    # Calculate vector b (matrix B dot product with present temperatures)
    b = np.zeros(space_divisions)
    b[1:-1] = B[1:-1, :].dot(T)

    # adjust vector for the front boundary condition
    if bc_surface_type == "linear":
        b[0] = 2*upsilon*T[1] + (1 - 2*upsilon - upsilon*2*dx*h/k)*T[0] + 4*upsilon*dx*h*T_air/k + \
            2*dx*upsilon/k * (heat_flux[j+1]+heat_flux[j])

    elif bc_surface_type == "mon-linear":
        b[0] = 2*upsilon*T[1] + (1- 2*upsilon - 2*dx*hc*upsilon/k)*T[0] + 4*dx*hc*upsilon*T_air/k + \
            4*emissivity*sigma*dx*upsilon*T[0]**4/k + 2*dx*upsilon/k * (heat_flux[j+1]+heat_flux[j])

    # adjust vector for the back boundary condition
    b[-1] = (1 - 2*upsilon)*T[-1] + 2*upsilon*T[-2]

    return b


'''
@Description:
@version:
@License: MIT
@Author: Wang Yao
@Date: 2020-03-23 19:42:15
@LastEditors: Wang Yao
@LastEditTime: 2020-03-27 17:50:33
'''
import os
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import Callback
from layers import PositionEncoding
from layers import MultiHeadAttention, PositionWiseFeedForward
from layers import Add, LayerNormalization

class Transformer(tf.keras.layers.Layer):

    def __init__(self, vocab_size, model_dim,
            n_heads=8, encoder_stack=6, decoder_stack=6, feed_forward_size=2048, dropout_rate=0.1, **kwargs):
        self._vocab_size = vocab_size
        self._model_dim = model_dim
        self._n_heads = n_heads
        self._encoder_stack = encoder_stack
        self._decoder_stack = decoder_stack
        self._feed_forward_size = feed_forward_size
        self._dropout_rate = dropout_rate
        super(Transformer, self).__init__(**kwargs)

    def build(self, input_shape):
        self.embeddings = self.add_weight(
            shape=(self._vocab_size, self._model_dim),
            initializer='glorot_uniform',
            trainable=True,
            name="embeddings")
        self.EncoderPositionEncoding = PositionEncoding(self._model_dim)
        self.EncoderMultiHeadAttetions = [
            MultiHeadAttention(self._n_heads, self._model_dim // self._n_heads)
            for _ in range(self._encoder_stack)
        ]
        self.EncoderLayerNorms0 = [
            LayerNormalization()
            for _ in range(self._encoder_stack)
        ]
        self.EncoderPositionWiseFeedForwards = [
            PositionWiseFeedForward(self._model_dim, self._feed_forward_size)
            for _ in range(self._encoder_stack)
        ]
        self.EncoderLayerNorms1 = [
            LayerNormalization()
            for _ in range(self._encoder_stack)
        ]
        self.DecoderPositionEncoding = PositionEncoding(self._model_dim)
        self.DecoderMultiHeadAttetions0 = [
            MultiHeadAttention(self._n_heads, self._model_dim // self._n_heads, future=True)
            for _ in range(self._decoder_stack)
        ]
        self.DecoderLayerNorms0 = [
            LayerNormalization()
            for _ in range(self._decoder_stack)
        ]
        self.DecoderMultiHeadAttetions1 = [
            MultiHeadAttention(self._n_heads, self._model_dim // self._n_heads)
            for _ in range(self._decoder_stack)
        ]
        self.DecoderLayerNorms1 = [
            LayerNormalization()
            for _ in range(self._decoder_stack)
        ]
        self.DecoderPositionWiseFeedForwards = [
            PositionWiseFeedForward(self._model_dim, self._feed_forward_size)
            for _ in range(self._decoder_stack)
        ]
        self.DecoderLayerNorms2 = [
            LayerNormalization()
            for _ in range(self._decoder_stack)
        ]
        super(Transformer, self).build(input_shape)

    def encoder(self, inputs):
        if K.dtype(inputs) != 'int32':
            inputs = K.cast(inputs, 'int32')

        masks = K.equal(inputs, 0)
        # Embeddings
        embeddings = K.gather(self.embeddings, inputs)
        embeddings *= self._model_dim ** 0.5 # Scale
        # Position Encodings
        position_encodings = self.EncoderPositionEncoding(embeddings)
        # Embedings + Postion-encodings
        encodings = embeddings + position_encodings
        # Dropout
        encodings = K.dropout(encodings, self._dropout_rate)

        for i in range(self._encoder_stack):
            # Multi-head-Attention
            attention = self.EncoderMultiHeadAttetions[i]
            attention_input = [encodings, encodings, encodings, masks]
            attention_out = attention(attention_input)
            # Add & Norm
            attention_out += encodings
            attention_out = self.EncoderLayerNorms0[i](attention_out)
            # Feed-Forward
            ff = self.EncoderPositionWiseFeedForwards[i]
            ff_out = ff(attention_out)
            # Add & Norm
            ff_out += attention_out
            encodings = self.EncoderLayerNorms1[i](ff_out)

        return encodings, masks

    def decoder(self, inputs):
        decoder_inputs, encoder_encodings, encoder_masks = inputs
        if K.dtype(decoder_inputs) != 'int32':
            decoder_inputs = K.cast(decoder_inputs, 'int32')

        decoder_masks = K.equal(decoder_inputs, 0)
        # Embeddings
        embeddings = K.gather(self.embeddings, decoder_inputs)
        embeddings *= self._model_dim ** 0.5 # Scale
        # Position Encodings
        position_encodings = self.DecoderPositionEncoding(embeddings)
        # Embedings + Postion-encodings
        encodings = embeddings + position_encodings
        # Dropout
        encodings = K.dropout(encodings, self._dropout_rate)

        for i in range(self._decoder_stack):
            # Masked-Multi-head-Attention
            masked_attention = self.DecoderMultiHeadAttetions0[i]
            masked_attention_input = [encodings, encodings, encodings, decoder_masks]
            masked_attention_out = masked_attention(masked_attention_input)
            # Add & Norm
            masked_attention_out += encodings
            masked_attention_out = self.DecoderLayerNorms0[i](masked_attention_out)

            # Multi-head-Attention
            attention = self.DecoderMultiHeadAttetions1[i]
            attention_input = [masked_attention_out, encoder_encodings, encoder_encodings, encoder_masks]
            attention_out = attention(attention_input)
            # Add & Norm
            attention_out += masked_attention_out
            attention_out = self.DecoderLayerNorms1[i](attention_out)

            # Feed-Forward
            ff = self.DecoderPositionWiseFeedForwards[i]
            ff_out = ff(attention_out)
            # Add & Norm
            ff_out += attention_out
            encodings = self.DecoderLayerNorms2[i](ff_out)

        # Pre-Softmax 与 Embeddings 共享参数
        linear_projection = K.dot(encodings, K.transpose(self.embeddings))
        outputs = K.softmax(linear_projection)
        return outputs

    def call(self, inputs):
        encoder_inputs, decoder_inputs = inputs
        encoder_encodings, encoder_masks = self.encoder(encoder_inputs)
        encoder_outputs = self.decoder([decoder_inputs, encoder_encodings, encoder_masks])
        return encoder_outputs

    def compute_output_shape(self, input_shape):
        return  (input_shape[0][0], input_shape[0][1], self._vocab_size)

class Noam(Callback):

    def __init__(self, model_dim, step_num=0, warmup_steps=4000, verbose=False, **kwargs):
        self._model_dim = model_dim
        self._step_num = step_num
        self._warmup_steps = warmup_steps
        self.verbose = verbose
        super(Noam, self).__init__(**kwargs)

    def on_train_begin(self, logs=None):
        logs = logs or {}
        init_lr = self._model_dim ** -.5 * self._warmup_steps ** -1.5
        K.set_value(self.model.optimizer.lr, init_lr)

    def on_batch_end(self, epoch, logs=None):
        logs = logs or {}
        self._step_num += 1
        lrate = self._model_dim ** -.5 * K.minimum(self._step_num ** -.5, self._step_num * self._warmup_steps ** -1.5)
        K.set_value(self.model.optimizer.lr, lrate)

    def on_epoch_begin(self, epoch, logs=None):
        if self.verbose:
            lrate = K.get_value(self.model.optimizer.lr)
            print(f"epoch {epoch} lr: {lrate}")

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        logs['lr'] = K.get_value(self.model.optimizer.lr)

def label_smoothing(inputs, epsilon=0.1):
    """目标平滑"""
    output_dim = inputs.shape[-1]
    smooth_label = (1 - epsilon) * inputs + (epsilon / output_dim)
    return smooth_label

if __name__ == "__main__":
    from tensorflow.keras.models import Model
    from tensorflow.keras.layers import Input
    from tensorflow.keras.utils import plot_model
    from tensorflow.keras.datasets import imdb
    from tensorflow.keras.preprocessing import sequence
    from tensorflow.keras.utils import to_categorical

    vocab_size = 5000
    max_seq_len = 256
    model_dim = 128

    print("Data downloading and pre-processing ... ")
    (x_train, y_train), (x_test, y_test) = imdb.load_data(maxlen=max_seq_len, num_words=vocab_size)
    x_train = sequence.pad_sequences(x_train, maxlen=max_seq_len)
    x_test = sequence.pad_sequences(x_test, maxlen=max_seq_len)
    x_train_masks = tf.equal(x_train, 0)
    x_test_masks = tf.equal(x_test, 0)
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)

    encoder_inputs = Input(shape=(max_seq_len,), name='encoder_inputs')
    decoder_inputs = Input(shape=(max_seq_len,), name='decoder_inputs')
    outputs = Transformer(vocab_size, model_dim)([encoder_inputs, decoder_inputs])
    outputs = tf.keras.layers.GlobalAveragePooling1D()(outputs)
    outputs = tf.keras.layers.Dense(2, activation='softmax')(outputs)
    model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=outputs)

    model.compile(
        loss=tf.keras.losses.categorical_crossentropy,
        optimizer='Adam',
        metrics=['accuracy']
    )

    model.fit(
        [x_train, x_train],
        y_train
    )


def factorial(x):

  if(x == 0 or x == 1):
      factored = 1
      return factored
  else:
    int(x)
  fact = 1
  for n in range(x,1,-1):
       fact*= n
       n-=1

  return fact

print(factorial(0))


# coding: utf-8

import math
from collections import OrderedDict
import random
import subprocess
import time
import psutil
import inspect
import os
import glob
import sys
import re
import random
import string
import collections
import socket
from random import randint
from operator import itemgetter
import csv
import json
from collections import OrderedDict
import hashlib
import unicodedata
import random
import re
import string
from systemtools.location import enhanceDir, getExecDir
from datetime import datetime
import parsedatetime
import arrow
from enum import Enum
from pytz import timezone
import pytz
from tzlocal import get_localzone
from dateutil import tz
from dateutil.tz import tzlocal

def stripAllLines(text, removeBlank=True):
    if text is None or not isinstance(text, str) or text == "":
        return text
    else:
        newText = ""
        for row in text.splitlines():
            current = row.strip()
            if removeBlank and current == "":
                pass
            else:
                newText += current + "\n"
        return newText

def timestampToDate(timestamp):
    return datetime.fromtimestamp(
        int(timestamp)
    ).strftime('%Y-%m-%d %H:%M:%S')

def linearScore(x, x1=0.0, x2=1.0, y1=0.0, y2=1.0, stayBetween0And1=True):
    a = (y2 - y1) / (x2 - x1)
    b = y1 - a * x1
    y = a * x + b
    if stayBetween0And1:
        if y <= 0.0:
            y = 0.0
        if y >= 1.0:
            y = 1.0
    return y

def camelCaseToUnderscoreCase(name):
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def camelCaseToUnderscoreCaseDict(theDict):
    newDict = {}
    for key, value in theDict.items():
        key = camelCaseToUnderscoreCase(key)
        newDict[key] = value
    return newDict

def reduceDictStr(theDict, max=40, replaceNewLine=False, reduceLists=False, maxElementCountInLists=1):
    if reduceLists and isinstance(theDict, list):
        values = theDict[:maxElementCountInLists]
        newValues = []
        for current in values:
            current = reduceDictStr(current,
                                    max=max,
                                    replaceNewLine=replaceNewLine,
                                    reduceLists=reduceLists,
                                    maxElementCountInLists=maxElementCountInLists)
            newValues.append(current)
        return newValues
    if not isinstance(theDict, dict):
        return theDict
    newDict = {}
    for key, value in theDict.items():
        if isinstance(value, str):
            if replaceNewLine:
                value = value.replace("\n", " ").replace("\r", " ")
            if len(value) > max:
                value = value[0:max-1]
        else:
            value = reduceDictStr(value,
                                    max=max,
                                    replaceNewLine=replaceNewLine,
                                    reduceLists=reduceLists,
                                    maxElementCountInLists=maxElementCountInLists)
        newDict[key] = value
    return newDict

def intersection(lists):
    """
        lists must contain lists of elements or a lists of strings then strings.
    """
    if lists is None or len(lists) == 0 or lists[0] is None:
        return []
    elif len(lists) == 1:
        return lists[0]
    else:
        theIntersection = list(lists[0])
        for current in lists[1:]:
            if current is None:
                theIntersection = []
                break
            else:
                if isinstance(current, str):
                    currentInter = []
                    for el in theIntersection:
                        if el in current:
                            currentInter.append(el)
                    theIntersection = currentInter
                    theIntersection = list(set(theIntersection))
                else:
                    theIntersection = list(set(current).intersection(theIntersection))
        return theIntersection

def strip(text):
    if text is None or not isinstance(text, str):
        return None
    else:
        return text.strip()

def lower(text):
    if text is None or not isinstance(text, str):
        return None
    else:
        return text.lower()

def askContinue(message=None, allowExit=True):
    if message is not None:
        print(message)
    toContinue = input("Continue? (Y/n)\n")
    toContinue = toContinue.lower().strip()
    if toContinue != "" and toContinue != "y":
        if allowExit:
            exit()
        else:
            return False
    return True

def dictContains(theDict, key):
    if theDict is None or key is None:
        return False
    if isinstance(theDict, str):
        theDict, key = key, theDict
    if key in theDict and theDict[key] is not None:
        return True
    return False

def dictContainsStr(key, theDict):
    if isinstance(theDict, str):
        theDict, key = key, theDict
    if key in theDict \
    and theDict[key] is not None \
    and isinstance(theDict[key], str) \
    and theDict[key].strip() != "":
        return True
    return False

def getObjectSize(*args, **kwargs):
    return objectSize(*args, **kwargs)
def objectSize(obj, readable=True):
    def sizeof_fmt(num, suffix='B'):
        for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
            if abs(num) < 1024.0:
                return "%3.1f%s%s" % (num, unit, suffix)
            num /= 1024.0
        return "%.1f%s%s" % (num, 'Yi', suffix)
    if readable:
        return sizeof_fmt(sys.getsizeof(obj))
    else:
        return sys.getsizeof(obj)

def hash(text):
    return md5(text)

def md5(text):
    theHash = None
    try:
        theHash = hashlib.md5(text)
    except TypeError:
        theHash = hashlib.md5(text.encode("utf-8"))
    else: pass
    if theHash is None:
        return None
    else:
        return theHash.hexdigest()

def dictFirstElement(theDict):
    if not isinstance(theDict, dict):
        return None
    if theDict is None:
        return None
    for key, value in theDict.items():
        return (key, value)

def dictRandomElement(theDict):
    if not isinstance(theDict, dict):
        return None
    if theDict is None:
        return None
    if len(theDict) == 0:
        return None
    randomIndex = getRandomInt(0, len(theDict) - 1)
    currentIndex = 0
    for key, value in theDict.items():
        if currentIndex == randomIndex:
            return (key, value)
        currentIndex += 1

def timestampToArrow(timestampInSec):
    return arrow.get(int(float(timestampInSec)))

def timestampMsToArrow(timestampInMs):
    return arrow.get(int(float(timestampInMs) / 1000.0))

def isDateStr(dateStr):
    try:
        arrow.get(dateStr)
        return True
    except:
        return False

def varname(p):
    for line in inspect.getframeinfo(inspect.currentframe().f_back)[3]:
        m = re.search(r'\bvarname\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)', line)
        if m:
            return m.group(1)

def getRandomName(addInt=True, maxInt=100):
    path = getExecDir(__file__) + "/data/fr-names.txt"
    names = None
    with open(path, 'r') as f:
        names = f.readlines()
    for i in range(len(names)):
        names[i] = names[i].strip()
    if names is None or len(names) == 0:
        return getRandomStr()
    name = random.choice(names)
    if addInt:
        name = name + "-" + str(getRandomInt(0, maxInt))
    return name

def getRandomStr(digitCount=10, withTimestamp=True):
    result = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(digitCount))
    if withTimestamp:
        timestamp = str(time.time())
        timestamp = timestamp.replace(".", "")
        result = result + "-" + timestamp
    return result

def getRandomFloat(min=0.0, max=1.0, decimalMax=2):
    """
        Warning:
            # This will give you [568, 905, 1114, 882, 1120, 899, 1074, 859, 1126, 900, 553] because 0 is a result of getRandomFloat() from 0.0 to 0.5, 10 is the result of 9.5 to 10.0
            count = [0] * 11
            for i in range(10000):
                current = int(round(getRandomFloat() * 10.0))
                count[current] += 1
            print(count)
            # So the solution is current = randint(0, 10) or:
            count = [0] * 11
            for i in range(10000):
                current = int(round(getRandomFloat() * 11.0 - 0.5))
                count[current] += 1
            print(count)
    """
    return round(random.uniform(min, max), decimalMax)

class Random:
    def __init__(self):
        self.nextSeed = None

    def setSeed(self, seed):
        self.nextSeed = self.getRandomInt()
        random.seed(seed)

    def resetSeed(self):
        random.seed(self.nextSeed)
        self.nextSeed = None

    def getRandomInt(self, number1=None, number2=None, seed=None, count=1):
        if number1 is None:
            number1 = sys.maxsize
        allRandomInts = []
        if seed is not None:
            nextSeed = getRandomInt(sys.maxsize)
            random.seed(seed)
        if number2 is None:
            theMin = 0
            theMax = number1
        else:
            theMin = number1
            theMax = number2
        for _ in range(count):
            current = random.randint(theMin, theMax)
            allRandomInts.append(current)
        if seed is not None:
            random.seed(nextSeed)
        if len(allRandomInts) == 1:
            return allRandomInts[0]
        else:
            return allRandomInts

    def getRandomBool(self):
        return bool(random.getrandbits(1))

    def getRandomFloat(self, min=0.0, max=1.0, decimalMax=6):
        return round(random.uniform(min, max), decimalMax)

def getRandomInt(number1=None, number2=None, seed=None, count=1):
    if number1 is None:
        number1 = sys.maxsize
    allRandomInts = []
    if seed is not None:
        nextSeed = getRandomInt(sys.maxsize)
        random.seed(seed)
    if number2 is None:
        theMin = 0
        theMax = number1
    else:
        theMin = number1
        theMax = number2
    if isinstance(theMax, float):
        theMax = int(theMax)
    if isinstance(theMin, float):
        theMin = int(theMin)
    for _ in range(count):
        current = random.randint(theMin, theMax)
        allRandomInts.append(current)
    if seed is not None:
        random.seed(nextSeed)
    if len(allRandomInts) == 1:
        return allRandomInts[0]
    else:
        return allRandomInts

def getRandomBool():
    return bool(random.getrandbits(1))

def isIterable(obj, returnTrueForNone=False):
    """
        # To test out of the systemtools-venv:
        isIterable(np.arange(0.0, 0.5, 0.1)

        :example:
        >>> isIterable([])
        True
        >>> isIterable([1])
        True
        >>> isIterable([None])
        True
        >>> isIterable(None)
        False
        >>> isIterable(None, returnTrueForNone=True)
        True
        >>> isIterable(1)
        False
        >>> isIterable({})
        True
        >>> isIterable("a")
        True
        >>> isIterable(random.randint)
        False
    """
    if obj is None:
        if returnTrueForNone:
            return True
        else:
            return False
    try:
        obj = iter(obj)
        return True
    except TypeError:
        return False

def getMinDict(theDict):
    return min(theDict, key=theDict.get)
def getMaxDict(theDict):
    return max(theDict, key=theDict.get)

def countNone(theList):
    return sum(x is None for x in theList)

def removeNone(theList):
    return [x for x in theList if x is not None]

def getFixedLengthQueue(count):
    return collections.deque(count * [None], count)

def dictToTuples(theDict):
    if isinstance(theDict, list):
        return theDict
    tupleList = []
    for key, value in theDict.items():
        tupleList.append((key, value))
    return tupleList

def tuplesToDict(tupleList):
    if isinstance(tupleList, dict):
        return tupleList
    theDict = {}
    for key, value in tupleList:
        theDict[key] = value
    return theDict

def maxTupleList(tupleList, index, getAll=True):
    # Check not none:
    if tupleList is None:
        return None
    # Remove None values:
    tupleList = [x for x in tupleList if x is not None]
    # Check length:
    if len(tupleList) == 0:
        return None
    # Return the max:
    if getAll:
        return max(tupleList, key=lambda item: item[index])
    else:
        return max(tupleList, key=itemgetter(index))[0]

def truncateFloat(f, n):
    '''Truncates/pads a float f to n decimal places without rounding'''
    s = '{}'.format(f)
    if 'e' in s or 'E' in s:
        return float('{0:.{1}f}'.format(f, n))
    i, p, d = s.partition('.')
    return float('.'.join([i, (d+'0'*n)[:n]]))

def listMean(l):
    return np.mean(l)

def normalize(theList):
    if theList is None:
        return theList
    else:
        return [float(i) / sum(theList) for i in theList]
#         return [float(i) / max(theList) for i in theList]

def partList(*args, **kwargs):
    return chunks(*args, **kwargs)
def chunks(*args, **kwargs):
    return list(chunksYielder(*args, **kwargs))
def chunksYielder(l, n):
    """Yield successive n-sized chunks from l."""
    if l is None:
        yield None
    elif len(l) <= 1:
        yield l
    elif n <= 1:
        yield l
    else:
        n = math.ceil(len(l) / n)
        if n >= len(l):
            for current in l:
                yield current
        else:
            for i in range(0, len(l), n):
                yield l[i:i + n]

def chunkList(l, partsCount):
    print("DEPRECATED, use chunks instead!")
    chunkedList = []
    partsItemNumber = int(math.ceil(float(len(l)) / float(partsCount)))
    for i in range(partsCount):
        left = l[:partsItemNumber]
        right = l[partsItemNumber:]
        chunkedList.append(left)
        l = right
    return chunkedList

def crossValidationChunk(l, partsCount):
    chunkedSet = chunkList(l, partsCount)
    trainingSets = []
    testSets = []
    for i in range(len(chunkedSet)):
        testSets.append(chunkedSet[i])
        currentMatrixTrainingSet = []
        for u in range(len(chunkedSet)):
            if u != i:
                currentMatrixTrainingSet.append(chunkedSet[u])
        currentListTrainingSet = []
        for current in currentMatrixTrainingSet:
            currentListTrainingSet += current
        trainingSets.append(currentListTrainingSet)
    return (trainingSets, testSets)

def sortBy(theDict, desc=False, index=1):
    data = theDict
    if isinstance(theDict, dict):
        data = theDict.items()
    return sorted(data, key=itemgetter(index), reverse=desc)

def sortByKey(theDict):
    return OrderedDict(sorted(theDict.items()))

def floatAsReadable(f):
    """
        source https://stackoverflow.com/questions/8345795/force-python-to-not-output-a-float-in-standard-form-scientific-notation-expo
    """
    _ftod_r = re.compile(br'^(-?)([0-9]*)(?:\.([0-9]*))?(?:[eE]([+-][0-9]+))?$')
    """Print a floating-point number in the format expected by PDF:
    as short as possible, no exponential notation."""
    s = bytes(str(f), 'ascii')
    m = _ftod_r.match(s)
    if not m:
        raise RuntimeError("unexpected floating point number format: {!a}"
                           .format(s))
    sign = m.group(1)
    intpart = m.group(2)
    fractpart = m.group(3)
    exponent = m.group(4)
    if ((intpart is None or intpart == b'') and
        (fractpart is None or fractpart == b'')):
        raise RuntimeError("unexpected floating point number format: {!a}"
                           .format(s))

    # strip leading and trailing zeros
    if intpart is None: intpart = b''
    else: intpart = intpart.lstrip(b'0')
    if fractpart is None: fractpart = b''
    else: fractpart = fractpart.rstrip(b'0')

    result = None

    if intpart == b'' and fractpart == b'':
        # zero or negative zero; negative zero is not useful in PDF
        # we can ignore the exponent in this case
        result = b'0'

    # convert exponent to a decimal point shift
    elif exponent is not None:
        exponent = int(exponent)
        exponent += len(intpart)
        digits = intpart + fractpart
        if exponent <= 0:
            result = sign + b'.' + b'0'*(-exponent) + digits
        elif exponent >= len(digits):
            result = sign + digits + b'0'*(exponent - len(digits))
        else:
            result = sign + digits[:exponent] + b'.' + digits[exponent:]

    # no exponent, just reassemble the number
    elif fractpart == b'':
        result = sign + intpart # no need for trailing dot
    else:
        result = sign + intpart + b'.' + fractpart

    return result.decode("utf-8")

def byteToStr(data):
    try:
        data = data.decode("utf-8")
    except AttributeError:
        pass
    return data

def removeNonASCII(str):
    printable = set(string.printable);
    return [x for x in str if x in printable];

def strListToHashCode(strList):
    return hashlib.md5("".join(strList)).hexdigest();

def strToHashCode(str):
#     str = str.encode("utf-8")
    return hashlib.md5(str).hexdigest()

def strToHashCode2(str):
    return hashlib.md5(str).hexdigest()

def trim(str):
    pattern = re.compile(r'\s+')
    str = re.sub(pattern, ' ', str)
    str = str.strip();
    return str

def isDigit(str):
    return str.isdigit()

def representsFloat(str):
    try:
        float(str)
        return True
    except ValueError:
        return False

def isFloat(str):
    try:
        float(str)
        return True
    except ValueError:
        return False

def isInt(str):
    try:
        int(str)
        return True
    except ValueError:
        return False

def isNumberList(l):
    for el in l:
        if not (isinstance(el, int) or isinstance(el, float)):
            return False
    return True

def isBooleanList(l):
    for el in l:
        if not (isinstance(el, bool)):
            return False
    return True

def representsInt(s, acceptRoundedFloats=False):
    """
        This function return True if the given param (string or float) represents a int

        :Example:
        >>> representsInt(1)
        True
        >>> representsInt("1")
        True
        >>> representsInt("a")
        False
        >>> representsInt("1.1")
        False
        >>> representsInt(1.1)
        False
        >>> representsInt(42.0, acceptRoundedFloats=True)
        True
        >>> representsInt("42.0", acceptRoundedFloats=True)
        True
    """

    if isinstance(s, float):
        if acceptRoundedFloats:
            return s.is_integer()
    else:
        if acceptRoundedFloats:
            try:
                s = float(s)
                return representsInt(s, acceptRoundedFloats=acceptRoundedFloats)
            except ValueError:
                return False
        else:
            try:
                int(s)
                return True
            except ValueError:
                return False
    return False

def listToStr2(obj, indent=4):
    return json.dumps(obj, indent=indent, sort_keys=True)

def listToStr(l, depth=0):
    tabs = ""
    for i in range(depth):
        tabs += "\t"
    if isinstance(l, list) and (isNumberOrBoolList(l)):
        result = "["
        for i in range(len(l)):
            el = l[i]
            result += str(el)
            if i != len(l) - 1:
                result += ", "
        result += "]"
        return result
    if isinstance(l, dict):
        result = ""
        result += "{"
        i = 0
        for key, value in list(l.items()):
            result += "\n"
            result += tabs + "\t" + str(key) + ": "
            if (isinstance(value, list) or isinstance(value, dict)) and not isNumberOrBoolList(value):
                result += "\n" + tabs + "\t"
            result += listToStr(value, depth + 1)
            if (i + 1) < len(l):
                result += ','
            i += 1
        result += "\n" + tabs + "}"
        return result
    elif isinstance(l, list):
        result = ""
        result += "["
        i = 0
        for el in l:
            result += "\n"
            result += tabs + "\t" + listToStr(el, depth + 1)
            if (i + 1) < len(l):
                result += ','
            i += 1
        result += "\n" + tabs + "]"
        return result
    else:
        if isinstance(l, str):
            return '"' + l + '"'
        else:
            return str(l)

def printLTS(l, *args, loggerFunct=None, **kwargs):
    result = listToStr(l, *args, **kwargs)
    if loggerFunct is not None:
        loggerFunct(result)
    else:
        print(result)

def isNumberOrBoolList(l):
    if len(l) == 0:
        return True
    for el in l:
        if not (isinstance(el, int) or isinstance(el, float) or isinstance(el, bool) or el is None):
            return False
    return True

def dictOfListToListOfDict(data):
    allKeys = list(data.keys())
    newData = []
    listLength = len(data[allKeys[0]])
    for currentIndex in range(listLength):
        newData.append(OrderedDict())
    for currentIndex in range(listLength):
        for currentKey in allKeys:
            newData[currentIndex][currentKey] = data[currentKey][currentIndex]
    return newData

def listOfDictToDictOfList(data):
    newData = OrderedDict()
    for key, value in list(data[0].items()):
        newData[key] = []
    for current in data:
        for key, value in list(current.items()):
            newData[key].append(value)
    return newData

def csvToList(filename, delimiter=';', quotechar='"', quoting=csv.QUOTE_NONNUMERIC, folder="./data/", rowLimit=None):
    folder = enhanceDir(folder)
    with open(folder + filename + '.csv', "r") as file:
        c = csv.reader(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
        keys = None
        data = []
        rowCount = 0
        for row in c:
            if rowLimit is not None and rowCount > rowLimit:
                break
            currentDict = None
            if keys is None:
                keys = row
            else:
                rowCount += 1
                currentDict = OrderedDict()
                for currentIndex in range(len(keys)):
                    currentDict[keys[currentIndex]] = row[currentIndex]
                data.append(currentDict)
        return data
    return None

def dictOrListToCSV(filename, data, delimiter=';', quotechar='"', quoting=csv.QUOTE_NONNUMERIC, folder="./output/"):
    """
    With these default params, all texts will be quoted. The delimiter is a ';'.
    If there are delimiters in a text, it will be escaped by using 2 double-quotes ('""').
    For example in calc, just set ';' as the column delimiter and '"' as the text delimiter.
    """
    folder = enhanceDir(folder)
    if not isinstance(data, list):
        data = dictOfListToListOfDict(data)
    with open(folder + filename + '.csv', "w") as file:
        c = csv.writer(file, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
        c.writerow(list(data[0].keys()))
        for currentRow in data:
            currentData = []
            for key, currentCol in list(currentRow.items()):
                currentData.append(currentCol)
            c.writerow(currentData)

        # w = csv.DictWriter(f, my_dict.keys())
    # w.writeheader()
    # w.writerow(my_dict)

    # with open(filename + '.csv', "wb") as f:
        # keys = data[0].keys()
            #  dict_writer = csv.DictWriter(f, keys)
        # dict_writer.writeheader()
        # dict_writer.writerows(data)

def stripAccents(s):
    if isinstance(s, str):
        return ''.join(c for c in unicodedata.normalize('NFD', s)
                  if unicodedata.category(c) != 'Mn')
    else:
        return s

def reduceStr(a,
              removeNumbers=False,
              toLowerCase=True,
              removeAccents=True,
              removePunct=True):
    """
    This funct convert u"-TTé01 fd-hb  /;\\      jdà  \n " to "TTe01 fd hb jda" for exemple
    """
    # if removePointAndComma:
        # a = re.sub(u'[,.]', u' ', a, flags=re.UNICODE)
    if removeAccents:
        a = stripAccents(a)
    if toLowerCase:
        a = a.lower()
    if removeNumbers:
        a = removeAllNumbers(a)

    # remove punct :
    if removePunct:
        # We search all ",1" or ".9"
        allFloatingPoint = re.finditer("[.,]\d", a)
        floatingPointIndex = []
        i = 0
        # We make a replacment list :
        for currentFloatingPoint in allFloatingPoint:
            floatingPointIndex.append\
            (
                ("opkfp" + str(i), currentFloatingPoint.group(0))
            )
            i += 1
        # but ".9" must be replaced by "\.9"
        for i in range(len(floatingPointIndex)):
            floatingPointIndex[i] = (floatingPointIndex[i][0], re.sub('[.]', '\\.', floatingPointIndex[i][1]))
        # We replace all :
        for tag, floatingPoint in floatingPointIndex:
            a = re.sub(floatingPoint, tag, a)
        # We delete punct :
        a = re.sub('[.,;?:!+=\-/_]', ' ', a)
        # And now we have to replace "\.9" by ".9" :
        for i in range(len(floatingPointIndex)):
            floatingPointIndex[i] = (floatingPointIndex[i][0], re.sub('\\\.', '.', floatingPointIndex[i][1]))
        # Now we re-insert all floating points :
        for tag, floatingPoint in floatingPointIndex:
            a = re.sub(tag, floatingPoint, a)

    # remove anything else but ".,;?:!+=\-/_" :
    a = re.sub('[^\w.,;?:!+=\-/_]', ' ', a, flags=re.UNICODE)

    # reduce space :
    a = re.sub('\s+', ' ', a)
    a = a.strip()

    # return the result :
    return a

def addSpaceBeforeUpperCase(text):
    return re.sub('([a-z])([A-Z])', '\\1 \\2', text)

def countOverlap(parsed1, parsed2, ngram):
    print("countOverlap DEPRECATED, pls use NLPTools!")
    if len(parsed1) < ngram or len(parsed2) < ngram:
        return 0
    count = 0
    for i in range(len(parsed1) - (ngram - 1)):
        currentParsed1Ngram = parsed1[i:i + ngram]
        for i in range(len(parsed2) - (ngram - 1)):
            currentParsed2Ngram = parsed2[i:i + ngram]
            if currentParsed1Ngram == currentParsed2Ngram:
                count += 1
    return count

def deleteDuplicate(l):
    return list(set(l))

def removeCommasBetweenDigits(text):
    """
        :example:
        >>> removeCommasBetweenDigits("sfeyv dsf,54dsf ef 6, 6 zdgy 6,919 Photos and 3,3 videos6,")
        'sfeyv dsf,54dsf ef 6, 6 zdgy 6919 Photos and 33 videos6,'
    """
    if text is None:
        return None
    else:
        return re.sub(r"([0-9]),([0-9])", "\g<1>\g<2>", text)

def getAllNumbers(text, removeCommas=False):
    if text is None:
        return None
    if removeCommas:
        text = removeCommasBetweenDigits(text)
    allNumbers = []
    if len(text) > 0:
        # Remove space between digits :
        spaceNumberExists = True
        while spaceNumberExists:
            text = re.sub('(([^.,0-9]|^)[0-9]+) ([0-9])', '\\1\\3', text, flags=re.UNICODE)
            if re.search('([^.,0-9]|^)[0-9]+ [0-9]', text) is None:
                spaceNumberExists = False
        numberRegex = '[-+]?[0-9]+[.,][0-9]+|[0-9]+'
        allMatchIter = re.finditer(numberRegex, text)
        if allMatchIter is not None:
            for current in allMatchIter:
                currentFloat = current.group()
                currentFloat = re.sub("\s", "", currentFloat)
                currentFloat = re.sub(",", ".", currentFloat)
                currentFloat = float(currentFloat)
                if currentFloat.is_integer():
                    allNumbers.append(int(currentFloat))
                else:
                    allNumbers.append(currentFloat)
    return allNumbers

def removeAllNumbers(text):
    if text is None:
        return None
    if len(text) == 0:
        return ""
    # Remove space between digits :
    spaceNumberExists = True
    while spaceNumberExists:
        text = re.sub('([0-9]) ([0-9])', '\\1\\2', text, flags=re.UNICODE)
        if re.search('[0-9] [0-9]', text) is None:
            spaceNumberExists = False
    numberRegex = '[-+]?[0-9]+[.,][0-9]+|[0-9]+'
    numberExists = True
    while numberExists:
        text = re.sub(numberRegex, "", text)
        if re.search(numberRegex, text) is None:
            numberExists = False

    return text.strip()

def getFirstNumber(text, *args, **kwargs):
    result = getAllNumbers(text, *args, **kwargs)
    if result is not None and len(result) > 0:
        return result[0]
    return None

def strCheck(value):
    return value is not None and isinstance(value, str) and len(value) > 0

def mergeDicts(*dict_args):
    """
    http://stackoverflow.com/questions/38987/how-to-merge-two-python-dictionaries-in-a-single-expression
    Given any number of dicts, shallow copy and merge into a new dict,
    precedence goes to key value pairs in latter dicts.
    """
    result = {}
    for dictionary in dict_args:
        if dictionary is not None:
            result.update(dictionary)
    return result

DATE_FORMAT = Enum("DATE_FORMAT", "datetimeString datetime timestamp arrow arrowString humanize")
def convertDate(readableDate=None, dateFormat=DATE_FORMAT.datetime):
    """
        Warning : utc shift may appear...
        Warning datetime.utcnow() return the utc+0 time
        datetime.now() return the current time in utc+2
    """
    if readableDate is None:
#         print('readableDate is None')
        theDate = datetime.now()
    elif isinstance(readableDate, datetime):
#         print('isinstance(readableDate, datetime)')
        theDate = readableDate
    elif isFloat(readableDate):
#         print('isFloat(readableDate)')
        theDate = datetime.fromtimestamp(float(readableDate))
    elif isDateStr(readableDate):
#         print('isDateStr(readableDate)')
        theDate = arrow.get(readableDate).datetime
    elif isinstance(readableDate, str):
#         print('isinstance(readableDate, str)')
        cal = parsedatetime.Calendar()
#         local_tz = get_localzone()
#         time_struct, parse_status = cal.parseDT(readableDate, tzinfo=local_tz)
        time_struct, parse_status = cal.parse(readableDate)
        theDate = datetime(*time_struct[:6])
    else:
        return None
#     local_tz = get_localzone()
#     print(local_tz)
#     local_now = theDate.replace(tzinfo=pytz.utc).astimezone(local_tz)
#     theDate = local_now
    timestamp = theDate.timestamp()
    if dateFormat == DATE_FORMAT.datetime:
        return theDate
    elif dateFormat == DATE_FORMAT.timestamp:
        return timestamp
    elif dateFormat == DATE_FORMAT.datetimeString:
        return str(theDate)
    elif dateFormat == DATE_FORMAT.humanize:
        return arrow.get(theDate).humanize() # locale='fr_fr'
    elif dateFormat == DATE_FORMAT.arrow:
        return timestampToArrow(timestamp)
    elif dateFormat == DATE_FORMAT.arrowString:
        return str(timestampToArrow(timestamp))
    else:
        return None

def test1():
    for readableDate in [
                        None,
                        datetime.now(),
                        datetime.now().timestamp(),
                        1506813088.541,
                        "1506813088",
                        "2017-10-22 14:11:10.800615",
                        "2017-10-22T14:11:10+00:00",
                        "2 mins ago",
                        "None",
                        "2017-10-01 01:23:35",
                        "aaa",
                        "2017-09-30T23:11:28+00:00",
                        "2 days ago", "in 2 days", "in 40 years",
                         "tomorrow at 6am", "next moday at noon",
                        "2 min ago", "3 weeks ago", "1 month ago",
                        "in 40 years at the 1st january 11am",
                        "1506813088.0",
                        1506813088.45247, "1506813088.45247",
                        1506813088,
                        "iusqhdfusdgfb jsdf dsgf usdbg",
                        ]:
        print(readableDate)
        for currentDateFormat in DATE_FORMAT:
            print(str(convertDate(readableDate, dateFormat=currentDateFormat)) + "\t\t" + str(currentDateFormat))
        print()
        print()
def test2():
        dateObj = datetime.now()
        dateObj = 'now'
        print(convertDate(dateObj, dateFormat=DATE_FORMAT.datetime))
        print(convertDate(dateObj, dateFormat=DATE_FORMAT.humanize))

def testPartList():
    lists = \
    [
#         [],
#         None,
#         [1],
#         [1, 2, 3],
        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
    ]
    for current in lists:
        printLTS(list(partList(current, 0)))
        printLTS(list(partList(current, 1)))
        printLTS(list(partList(current, 2)))
        printLTS(list(partList(current, 3)))
        printLTS(list(partList(current, 4)))

if __name__ == '__main__':
#     print(normalize([0.5, 0.5, 1.0, 2.0]))
#     print(normalize([0.2, 0.2, 0.4, 0.2]))
#     print(normalize([20, 20, 40, 20]))
#     print(normalize([10, 30, 40, 50]))
#     print(normalize([0.1, 0.1, 0.2]))
#     test1()
#     test2()
#     print(datetime.now())
#     testPartList()

#
#     dicts = \
#     [
#         {"a": 1, "b": 2},
#         {"b": 2, "a": 1},
#         {"a": 1},
#         {},
#         None,
#         {"b": 2, "a": 1, "c": 3},
#         {"b": 2, "a": 1, "c": 3, "d": 4},
#     ]
#
#
#     for i in range(1000):
#         for currentDict in dicts:
#             printLTS(dictRandomElement(currentDict))
#

#     for i in range(100):
#         print(getRandomName())

#     theDict = {}
#     for i in range(1000):
#         theDict[getRandomInt(0, 1000000)] = getRandomInt(0, 1000000)
#     print(objectSize())
    test2()

def listSubstract(a, b):
    if a is None:
        return []
    elif b is None:
        return a
    else:
        return [item for item in a if item not in b]

# We don't need this because "test"[0:30] doesn't raise any exception...
# def cut(theString, minIndex, maxIndex=None):
#     """
#         Return the same string truncated in [minIndex, maxIndex]
#         These index (at the border) include the corresponding letter
#         If max is too long, it will be replaced by -1
#         If max is None, max become min et min become 0
#
#         :example:
#         >>> cut("test", 2)
#         "tes"
#         >>> cut("test", 2, -1)
#         "st"
#         >>> cut("test", 0, -1)
#         "test"
#         >>> cut("test", -2)
#         "tes"
#         >>> cut("test", -2, -1)
#         "st"
#         >>> cut(None, -2, -1)
#         None
#         >>> cut("", -2, -1)
#         ""
#         >>> cut(None, 1)
#         None
#         >>> cut("", 1)
#         ""
#     """
#     if maxIndex is None:
#         maxIndex = minIndex
#         minIndex = 0
#     theStringLen = len(theString)
#     if maxIndex > theStringLen:
#         maxIndex = theStringLen
#     return theString[minIndex:maxIndex]


from bs4 import BeautifulSoup

class Transforms:
    @classmethod
    def data_types(cls):
        return cls.convert_rules.keys()

    # If there is a rule defined here, the corresponding CamelCase model
    # will be indexed
    convert_rules = {
        "committee": {
            "id": "id",
            "title": "name",
            "description": ["info", "about"]
        },
        "committee_meeting": {
            "id": "id",
            "title": "title",
            "description": "summary",
            "fulltext": "body",
            "date": "date",
            "committee_name": ["organisation", "name"],
            "committee_id": ["organisation", "id"],
        },
        "member": {
            "id": "id",
            "title": "name",
            "description": "bio",
            "date": "start_date",
        },
        "bill": {
            "id": "id",
            "title": "title",
            "description": "bill_code"
        },
        "hansard": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
        },
        "briefing": {
            "id": "id",
            "title": "title",
            "description": "summary",
            "fulltext": "minutes",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "question_reply": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "tabled_committee_report": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "call_for_comment": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
            "committee_name": ["committee", "name"],
            "committee_id": ["committee", "id"],
        },
        "policy_document": {
            "id": "id",
            "title": "title",
            "date": "start_date",
        },
        "gazette": {
            "id": "id",
            "title": "title",
            "date": "start_date",
        },
        "daily_schedule": {
            "id": "id",
            "title": "title",
            "fulltext": "body",
            "date": "start_date",
        },
    }

    @classmethod
    def serialise(cls, data_type, obj):
        item = {}
        for key, field in Transforms.convert_rules[data_type].iteritems():
            val = cls.get_val(obj, field)
            if isinstance(val, unicode):
                val = BeautifulSoup(val).get_text().strip()
            item[key] = val
        return item

    @classmethod
    def get_val(cls, obj, field):
        if isinstance(field, list):
            if (len(field) == 1):
                if hasattr(obj, field[0]):
                    return getattr(obj, field[0])
            key = field[:1][0]
            if isinstance(key, int):
                if len(obj) > key:
                    return cls.get_val(obj[key], field[1:])
                return None
            if hasattr(obj, key):
                newobj = getattr(obj, key)
                return cls.get_val(newobj, field[1:])
            else:
                return None
        else:
            return getattr(obj, field)


import json

import requests

from django.conf import settings

from pyconca2017.pycon_schedule.models import Speaker, Presentation

class PapercallInterface(object):

    BASE_URL = 'https://www.papercall.io/api/v1'

    EVENT_URL = '/event'
    SUBMISSIONS_LIST_URL = '/submissions'
    SUBMISSION_GET_URL = '/submissions/{submission_id}'
    SUBMISSION_RATINGS_URL = '/submissions/{submission_id}/ratings'

    class SubmissionStates(object):
        ACCEPTED = 'accepted'
        SUBMITTED = 'submitted'
        REJECTED = 'rejected'
        WAITLIST = 'waitlist'

    def __init__(self):
        self.client = requests.Session()
        self.client.headers.update({'Authorization': settings.PAPERCALL_TOKEN})

    def get_submissions(self, state=SubmissionStates.ACCEPTED):
        """ Iterator """
        url = '{}{}'.format(self.BASE_URL, self.SUBMISSIONS_LIST_URL)
        params = {
            'per_page': 100,
            'page': 0,
            'order': 'created_at',
        }
        if state:
            params['state'] = state

        while True:
            params['page'] += 1
            response = self.client.get(url, params=params)

            response_pagination = json.loads(response.headers.get('x-pagination'))

            data = response.json()

            for item in data:
                yield item

            if response_pagination['last_page']:
                break

class PresentationService(object):

    def __init__(self):
        self.papercall = PapercallInterface()

    def sync_proposals(self, update=False):
        """
        Sync Papercall submissions with the database.

        :param update: If True, all values will be updated from Papercall.
        :return:
        """

        for submission in self.papercall.get_submissions():
            speaker_data = self._submission_to_speaker_data(submission)
            talk_data = self._submission_to_presentation_data(submission)

            speaker = self._sync_speaker(speaker_data, update=update)
            talk_data['speaker'] = speaker
            self._sync_presentation(talk_data, update=update)

    def _submission_to_speaker_data(self, submission):
        profile = submission['profile']
        return {
            'full_name': profile['name'],
            'bio': profile['bio'],
            'twitter_username': profile['twitter'],
            'company_name': profile['company'],
            'url': profile['url'],
            'shirt_size': profile['shirt_size'],
            'email': profile['email'],
            'location': profile['location'],
        }

    def _sync_speaker(self, speaker_data, update=False):
        if update:
            speaker = Speaker.objects.update_or_create(email=speaker_data['email'], defaults=speaker_data)[0]
        else:
            speaker = Speaker.objects.get_or_create(email=speaker_data.pop('email'), defaults=speaker_data)[0]

        return speaker

    def _submission_to_presentation_data(self, submission):
        talk = submission['talk']

        return {
            'papercall_id': submission['id'],
            'title': talk['title'],
            'description': talk['description'],
            'notes': talk['notes'],
            'abstract': talk['abstract'],
            'audience_level': talk['audience_level'],
            'presentation_format': talk['talk_format'],
        }

    def _sync_presentation(self, data, update=False):
        if update:
            presentation = Presentation.objects.update_or_create(papercall_id=data['papercall_id'], defaults=data)[0]
        else:
            presentation = Presentation.objects.get_or_create(papercall_id=data.pop('papercall_id'), defaults=data)[0]

        return presentation


# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

#!/usr/bin/env python2.7

"""Send JPEG image to tensorflow_model_server loaded with inception model.
"""

from __future__ import print_function

# This is a placeholder for a Google-internal import.

from grpc.beta import implementations
import tensorflow as tf
from PIL import Image, ImageFilter
import numpy as np
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2
import cv2
from protobuf_to_dict import protobuf_to_dict

tf.app.flags.DEFINE_string('server', 'localhost:9000',
                           'PredictionService host:port')
tf.app.flags.DEFINE_string('image', '', 'path to image in JPEG format')
FLAGS = tf.app.flags.FLAGS

def imageprepare(argv):
    im = Image.open(argv).convert('L')
    width = float(im.size[0])
    height = float(im.size[1])
    newImage = Image.new('L', (28, 28), (255))  # creates white canvas of 28x28 pixels

    if width > height:  # check which dimension is bigger
        # Width is bigger. Width becomes 20 pixels.
        nheight = int(round((20.0 / width * height), 0))  # resize height according to ratio width
        if (nheight == 0):  # rare case but minimum is 1 pixel
            nheight = 1
            # resize and sharpen
        img = im.resize((20, nheight), Image.ANTIALIAS).filter(ImageFilter.SHARPEN)
        wtop = int(round(((28 - nheight) / 2), 0))  # calculate horizontal position
        newImage.paste(img, (4, wtop))  # paste resized image on white canvas
    else:
        # Height is bigger. Heigth becomes 20 pixels.
        nwidth = int(round((20.0 / height * width), 0))  # resize width according to ratio height
        if (nwidth == 0):  # rare case but minimum is 1 pixel
            nwidth = 1
            # resize and sharpen
        img = im.resize((nwidth, 20), Image.ANTIALIAS).filter(ImageFilter.SHARPEN)
        wleft = int(round(((28 - nwidth) / 2), 0))  # caculate vertical pozition
        newImage.paste(img, (wleft, 4))  # paste resized image on white canvas

    # newImage.save("sample.png

    tv = list(newImage.getdata())  # get pixel values

    # normalize pixels to 0 and 1. 0 is pure white, 1 is pure black.
    tva = [(255 - x) * 1.0 / 255.0 for x in tv]
    return tva

def main(_):
  host, port = FLAGS.server.split(':')
  channel = implementations.insecure_channel(host, int(port))
  stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
  # Send request
  with open(FLAGS.image, 'rb') as f:
    # See prediction_service.proto for gRPC request/response details.
   # data=cv2.imread(FLAGS.image)
    x=[imageprepare(FLAGS.image)]
    data=np.array(x[0])
    request = predict_pb2.PredictRequest()
    request.model_spec.name = 'mnist'
    request.model_spec.signature_name = 'predict_images'
    request.inputs['images'].CopyFrom(
          tf.contrib.util.make_tensor_proto(data,shape=[1, data.size],dtype=tf.float32 ))
    result = stub.Predict(request, 10.0)  # 10 secs timeou
    for x in dict(result.outputs)['scores'].float_val :
       print(x)

if __name__ == '__main__':
  tf.app.run()


#Don't forget to add both 'ExecuteAPI' and 'System' (AWS Systems Manger) to your permissions as inline policys for this lambda function

import json
import boto3
Websocket_HTTPS_URL = "https://<Insert-Websocket-Endpoint_Here>"
client = boto3.client("apigatewaymanagementapi", endpoint_url = Websocket_HTTPS_URL)
ssm_Client = boto3.client('ssm')

def lambda_handler(event, context):
    print(event)
    response_ssm = ssm_Client.get_parameter(Name='<Insert-Your-SSM-Parameter-Name-Here>')
    print("my stored connection id: ", response_ssm['Parameter']['Value'] )
    connectionId =  response_ssm['Parameter']['Value']  #dig into the response blob to get our string cvalue
    Test_Message = json.dumps({ "message": "Hello from lambda, hardcoded test message"})
    IoT_Message = json.dumps(event)
    #AWS API Gateway API's require 'key=value' arguments
    #https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/apigatewaymanagementapi.html#ApiGatewayManagementApi.Client.post_to_connection
    response = client.post_to_connection(ConnectionId = connectionId, Data = IoT_Message)


"""Tests for rich module."""
import sys

from enrich.console import Console, should_do_markup

def test_rich_console_ex() -> None:
    """Validate that ConsoleEx can capture output from print() calls."""
    console = Console(record=True, redirect=True)
    console.print("alpha")
    print("beta")
    sys.stdout.write("gamma\n")
    sys.stderr.write("delta\n")
    # While not supposed to happen we want to be sure that this will not raise
    # an exception. Some libraries may still sometimes send bytes to the
    # streams, notable example being click.
    sys.stdout.write(b"epsilon\n")  # type: ignore
    text = console.export_text()
    assert text == "alpha\nbeta\ngamma\ndelta\nepsilon\n"

def test_rich_console_ex_ansi() -> None:
    """Validate that ANSI sent to sys.stdout does not become garbage in record."""
    print()
    console = Console(force_terminal=True, record=True, redirect=True)
    console.print("[green]this from Console.print()[/green]", style="red")

    text = console.export_text(clear=False)
    assert "this from Console" in text

    html = console.export_html(clear=False)
    assert "#008000" in html

def test_console_soft_wrap() -> None:
    """Assures long prints on console are not wrapped when requested."""
    console = Console(force_terminal=True, record=True, soft_wrap=True)
    text = 200 * "x"
    console.print(text)
    result = console.export_text()
    assert text in result

def test_console_print_ansi() -> None:
    """Validates that Console.print() with ANSI does not make break them."""
    console = Console(force_terminal=True, record=True, soft_wrap=True, redirect=True)
    text = "\033[92mfuture is green!\033[0m"
    console.print(text)
    text_result = console.export_text(clear=False)
    assert "future is green!" in text_result
    html_result = console.export_html()
    assert "#00ff00" in html_result

def test_markup_detection_pycolors0(monkeypatch):
    """Assure PY_COLORS=0 disables markup."""
    monkeypatch.setenv("PY_COLORS", "0")
    assert not should_do_markup()

def test_markup_detection_pycolors1(monkeypatch):
    """Assure PY_COLORS=1 enables markup."""
    monkeypatch.setenv("PY_COLORS", "1")
    assert should_do_markup()

def test_markup_detection_tty_yes(mocker):
    """Assures TERM=xterm enables markup."""
    mocker.patch("sys.stdout.isatty", return_value=True)
    mocker.patch("os.environ", {"TERM": "xterm"})
    assert should_do_markup()
    mocker.resetall()
    mocker.stopall()

def test_markup_detection_tty_no(mocker):
    """Assures that if no tty is reported we disable markup."""
    mocker.patch("os.environ", {})
    mocker.patch("sys.stdout.isatty", return_value=False)
    assert not should_do_markup()
    mocker.resetall()
    mocker.stopall()

if __name__ == "__main__":
    test_console_print_ansi()


from typing import List

def sum_of_two(nList: List[int], N: int) -> bool:
    #nList is a number list
    for i in range(0, len(nList)):
        for j in range(i+1, len(nList)):
            if nList[i] + nList[j] == N:
                return True
    return False

def main() -> None:
    lst = list(map(int, input().split()))
    N = int(input())
    print(sum_of_two(lst, N))

if __name__ == "__main__":
    main()


#!/usr/bin/python
#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

# Assumption: pip install keras
#
# This test validates SystemML's deep learning APIs (Keras2DML, Caffe2DML and nn layer) by comparing the results with that of keras.
#
# To run:
#   - Python 2: `PYSPARK_PYTHON=python2 spark-submit --master local[*] --driver-memory 10g --driver-class-path SystemML.jar,systemml-*-extra.jar test_nn_numpy.py`
#   - Python 3: `PYSPARK_PYTHON=python3 spark-submit --master local[*] --driver-memory 10g --driver-class-path SystemML.jar,systemml-*-extra.jar test_nn_numpy.py`

# Make the `systemml` package importable
import os
import sys
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")
sys.path.insert(0, path)

import unittest

import numpy as np
from keras.models import Sequential
from keras.layers import Input, Dense, Conv2D, MaxPooling2D, Dropout, Flatten, LSTM, UpSampling2D, SimpleRNN
from keras import backend as K
from keras.models import Model
from systemml.mllearn import Keras2DML
from pyspark.sql import SparkSession

batch_size = 32
input_shape = (3,64,64)
K.set_image_data_format("channels_first")
# K.set_image_dim_ordering("th")
keras_tensor = np.random.rand(batch_size,input_shape[0], input_shape[1], input_shape[2])
sysml_matrix = keras_tensor.reshape((batch_size, -1))
tmp_dir = 'tmp_dir'

spark = SparkSession.builder.getOrCreate()

def are_predictions_all_close(keras_model, rtol=1e-05, atol=1e-08):
    sysml_model = Keras2DML(spark, keras_model, input_shape=input_shape, weights=tmp_dir)
    keras_preds = keras_model.predict(keras_tensor).flatten()
    sysml_preds = sysml_model.predict_proba(sysml_matrix).flatten()
    #print(str(keras_preds))
    #print(str(sysml_preds))
    return np.allclose(keras_preds, sysml_preds, rtol=rtol, atol=atol)

class TestNNLibrary(unittest.TestCase):
    def test_1layer_upsample_predictions1(self):
        keras_model = Sequential()
        keras_model.add(UpSampling2D(size=(2, 2), input_shape=input_shape))
        keras_model.add(Flatten())
        keras_model.add(Dense(10, activation='softmax'))
        self.failUnless(are_predictions_all_close(keras_model, atol=1e-06))

    def test_1layer_upsample_predictions2(self):
        keras_model = Sequential()
        keras_model.add(UpSampling2D(size=(2, 3), input_shape=input_shape))
        keras_model.add(Flatten())
        keras_model.add(Dense(10, activation='softmax'))
        self.failUnless(are_predictions_all_close(keras_model, atol=1e-06))

    def test_1layer_cnn_predictions(self):
        keras_model = Sequential()
        keras_model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape, padding='valid'))
        keras_model.add(Flatten())
        keras_model.add(Dense(10, activation='softmax'))
        self.failUnless(are_predictions_all_close(keras_model))

    def test_multilayer_cnn_predictions(self):
        keras_model = Sequential()
        keras_model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape, padding='valid'))
        keras_model.add(MaxPooling2D(pool_size=(2, 2)))
        keras_model.add(Conv2D(64, (3, 3), activation='relu'))
        keras_model.add(MaxPooling2D(pool_size=(2, 2)))
        keras_model.add(Flatten())
        keras_model.add(Dense(256, activation='softmax'))
        keras_model.add(Dropout(0.25))
        keras_model.add(Dense(10, activation='softmax'))
        self.failUnless(are_predictions_all_close(keras_model))

    def test_simplernn_predictions1(self):
        data_dim = 16
        timesteps = 8
        num_classes = 10
        batch_size = 64
        model = Sequential()
        model.add(SimpleRNN(32, return_sequences=False, input_shape=(timesteps, data_dim)))
        model.add(Dense(10, activation='softmax'))
        x_train = np.random.random((batch_size, timesteps, data_dim))
        y_train = np.random.random((batch_size, num_classes))
        from systemml.mllearn import Keras2DML
        sysml_model = Keras2DML(spark, model, input_shape=(timesteps,data_dim,1), weights='weights_dir').set(debug=True)
        keras_preds = model.predict(x_train).flatten()
        sysml_preds = sysml_model.predict_proba(x_train.reshape((batch_size, -1))).flatten()
        self.failUnless(np.allclose(sysml_preds, keras_preds))

    def test_simplernn_predictions2(self):
        data_dim = 16
        timesteps = 8
        num_classes = 10
        batch_size = 100
        model = Sequential()
        model.add(SimpleRNN(32, return_sequences=False, input_shape=(timesteps, data_dim)))
        model.add(Dense(10, activation='softmax'))
        x_train = np.random.random((batch_size, timesteps, data_dim))
        y_train = np.random.random((batch_size, num_classes))
        from systemml.mllearn import Keras2DML
        sysml_model = Keras2DML(spark, model, input_shape=(timesteps,data_dim,1), weights='weights_dir').set(debug=True)
        keras_preds = model.predict(x_train).flatten()
        sysml_preds = sysml_model.predict_proba(x_train.reshape((batch_size, -1))).flatten()
        self.failUnless(np.allclose(sysml_preds, keras_preds))

    def test_lstm_predictions1(self):
        data_dim = 32
        timesteps = 8
        num_classes = 10
        batch_size = 64
        w1 = np.random.random((data_dim, 4*data_dim))
        w2 = np.random.random((data_dim, 4*data_dim))
        b = np.zeros(128)
        model = Sequential()
        model.add(LSTM(32, return_sequences=False, recurrent_activation='sigmoid', input_shape=(timesteps, data_dim),  weights=[w1, w2, b]))
        model.add(Dense(10, activation='softmax'))
        x_train = np.random.random((batch_size, timesteps, data_dim))
        y_train = np.random.random((batch_size, num_classes))
        from systemml.mllearn import Keras2DML
        sysml_model = Keras2DML(spark, model, input_shape=(timesteps,data_dim,1), weights='weights_dir').set(debug=True)
        keras_preds = model.predict(x_train)
        sysml_preds = sysml_model.predict_proba(x_train.reshape((batch_size, -1)))
        np.allclose(sysml_preds, keras_preds)

if __name__ == '__main__':
    unittest.main()


'''
Order statistics

find kth smallest

k=1
k=n
k=n+1/2

randomized divide and conquer

Random-Select(A, p, q, i)

ith smallest in the A[p, q]

if p = q then return A[p]
r <- Rand-Partitioan(A, p, q)
k <- r-p+1 // k = rank(A[r])

if i = k then return A[r]

if i < k then return Random-Select(A, p, r-1, i)

else  return Random-Select(A, r+1, q, i-k)
'''
from PSort import _partition

def randSelect(A, p, q, i):
    r = _partition(A, p, q)
    k = r - p + 1
    if  i == k:
        return A[r]
    if  i < k:
        return randSelect(A, p, r-1, i)
    else:
        return randSelect(A, r+1, q, i-k)

'''
worst case linear time order stats
good pivot force good good pivot

Select(i,n)
1, divide n element in to n/5 group of 5 elems each
   find middle of each group
2, recursively select the  middle of the n/5 group

3, partition with x of pivot let k = rank(x)

4, if i = k return x
   if i < k return  randSelect(.......)

   else return  randSeclt(.....)

'''


from typing import List
import logging

from src.app.AbstractDataUpdater import AbstractDataUpdater

from src.helpers.db import get_data_for_reg_window

import pandas as pd

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class ProductFunnelDataUpdater(AbstractDataUpdater):

	def _get_columns_to_upsert(self) -> List[str]:
		return [
			"date_reg", "campaign", "is_target", "registrations", "seen_search", "uploaded_photo", "visited_female",
			"took_credits", "spent_0", "spent_10", "message_sent", "platform",
		]

	def _get_constraints_to_upsert(self) -> List[str]:
		return [
			"date_reg", "campaign", "is_target", "platform",
		]

	def _get_table_name(self) -> str:
		return "product"

	def download_data(self, reference: dict):

		window: int = int(self.cw.get_key_from_section_of_config("env", "window"))

		analytics: pd.DataFrame = get_data_for_reg_window(
			reference["analytics"]["db"],
			reference["analytics"]["query"],
			window
		)

		product: pd.DataFrame = get_data_for_reg_window(
			reference["product"]["db"],
			reference["product"]["query"],
			window
		)

		users: pd.DataFrame = analytics.merge(product, how="left", left_on="id_user", right_on="id_user")

		self.data = users

	def process_data(self):

		self.data["campaign"] = self.data["campaign"].apply(
			lambda x: "AW_Rondevo_USA_UAC_NoCampaign" if (pd.isna(x) or x == "" or x == " ") else x
		)

		self.data["years"] = self.data["years"].apply(
			lambda x: int(0) if (pd.isna(x) or x == "" or x == " ") else int(x)
		)
		self.data["is_target"] = self.data["years"].apply(
			lambda x: 1 if x >= 35 else 0
		)
		self.data.drop("years", axis=1, inplace=True)

		self.data["credits_spent"] = self.data["credits_spent"].apply(lambda x: int(x) if not pd.isna(x) else int(0))

		constraints: list = self._get_constraints_to_upsert()

		self.data = self.data.groupby(constraints).apply(
			lambda x: pd.Series(
				dict(
					registrations=x["id_user"].nunique(),

					seen_search=x["seen_search"].sum(),
					uploaded_photo=x["uploaded_photo"].sum(),
					visited_female=x["visited_female"].sum(),

					took_credits=x["took_credits"].sum(),
					spent_0=x[x["credits_spent"] > 0]["id_user"].nunique(),
					spent_10=x[x["credits_spent"] >= 10]["id_user"].nunique(),

					message_sent=x["message_sent"].sum()

				)
			)
		).reset_index()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
https://leetcode.com/problems/two-sum-iv-input-is-a-bst/description/

Given a Binary Search Tree and a target number,
return true if there exist two elements in the BST such
that their sum is equal to the given target.

Example 1:
Input:
    5
   / \
  3   6
 / \   \
2   4   7

Target = 9

Output: True

Example 2:
Input:
    5
   / \
  3   6
 / \   \
2   4   7

Target = 28

Output: False
"""

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution(object):
    def findTarget(self, root, k):
        """
        :type root: TreeNode
        :type k: int
        :rtype: bool
        """

        def find(root, v):
            if root.val == v:
                return True

            if v < root.val and root.left:
                return find(root.left, v)
            elif root.right:
                return find(root.right, v)

            return False

        bfs_list = [root]

        while bfs_list:
            node = bfs_list.pop(0)
            remain = k - node.val

            if remain != node.val and find(root, remain):
                return True

            if node.left:
                bfs_list.append(node.left)
            if node.right:
                bfs_list.append(node.right)

        return False

def build():
    """
    9
   / \
  3  20
 /  /  \
2  15   22
    """
    root = TreeNode(1)
    return root, 2

    root = TreeNode(2)
    root.left = TreeNode(1)
    root.right = TreeNode(3)
    return root, 4

    root = TreeNode(9)
    root.left = TreeNode(3)
    root.left.left = TreeNode(2)
    root.right = TreeNode(20)
    root.right.right = TreeNode(22)
    root.right.left = TreeNode(15)
    return root, 12

if __name__ == "__main__":
    s = Solution()
    print(s.findTarget(*build()))


from ProjectColaborador import ProjectColaborador
from ProjectManager import ProjectManager
from dbadapter import DbAdapter

class ProjectAdmin(ProjectColaborador):
    def __init__(self,request):
        ProjectColaborador.__init__(self, request)

    @classmethod
    def delete_proyecto(cls, id_proyecto):
        adapter=DbAdapter()
        adapter.borrar_proyecto(id_proyecto)

    @classmethod
    def delete_pista(cls,id_proyecto,nombrePista):
        adapter=DbAdapter()
        adapter.borrar_pista(id_proyecto,nombrePista)

    @classmethod
    def modificar_proyecto(cls):
        request = cls.request
        if request.method == 'POST':
            id = request.form['id']
            proyecto = request.form['proyecto']
            imagen = request.files['imagen']
            descripcion = request.form['descripcion']
            if ProjectManager.allowed_image(imagen.filename):
                db.modificar_proyecto(id, proyecto, imagen, descripcion)
            else:
                return 'Alguno de los archivos subidos no tiene la extensión propicia.'

    # privado es true y publico false
    @classmethod
    def set_privacidad(cls, privacidad, proyecto):
        request = cls.request
        if request.method == 'POST':
            proyecto = request.form['id_proyecto']
            privacidad = request.form['privacidad']
            if privacidad == "publico" or privacidad == "privado":
                privado = 0 if "privado" == privacidad else 1
                db.update_privacidad_proyecto(privado, proyecto)
            else:
                return 'La privacidad puede ser publica o privada'


from src.logic import Logic, Fileworks

#region consts
EU, NA, KR, EVENT, ALL = "eu", "na", "kr", "event", "all"
#maps consts
TOD, TOTSQ, IS, CH, DS, BOE, ST, WJ, BH = "Towers_of_Doom", "Tomb_of_the_Spider_Queen", "Infernal_Shrines", \
        "Cursed_Hollow", "Dragon_Shire", "Battlefield_of_Eternity", "Sky_Temple", "Warhead_Junction", "Braxis_Holdout"

if __name__ == '__main__':
    def send(number, message):
        print("({}) {}".format(number, message))

    def show_teams(region):
        i = 1
        send(0, "Back")
        if region == EU:
            for team in logic.eu_teams:
                if i == 9:
                    send(i, "Next")
                    break
                send(i, team.name.replace('_', ' '))
                i += 1
        elif region == NA:
            for team in logic.na_teams:
                if i == 9:
                    send(i, "Next")
                    break
                send(i, team.name.replace('_', ' '))
                i += 1
        elif region == KR:
            for team in logic.kr_teams:
                if i == 9:
                    send(i, "Next")
                    break
                send(i, team.name.replace('_', ' '))
                i += 1
        else:
            logic.sort_teams()
            for team in logic.teams:
                if i == 9:
                    send(i, "Next")
                    break
                send(i, team.name.replace('_', ' '))
                i += 1

    print("Hi lol!")
    logic = Logic()
    fileworks = Fileworks("data/hgc_eu_results.txt", "data/hgc_na_results.txt", "data/hgc_kr_results.txt",
                          "data/hgc_eu_teams.txt", "data/hgc_na_teams.txt", "data/hgc_kr_teams.txt", logic)

    fileworks.load_results()
    # logic.parse_all_results()
    # logic.add_result("2017-02-12", DS, logic.find_team("Fnatic", EU), logic.find_team("Misfits", EU), 1, EU)
    fileworks.save_results()
    show_teams(EU)
    fileworks.save_teams()


__author__    = "Andre Merzky"
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

""" Unit tests for saga.utils.signatures
"""

import saga

def test_signatures () :
    """ Test if signature violations are flagged """

    try :
        s = saga.Session ('should not accept a string')
        assert False, "should have seen a BadParameter exception"
    except saga.BadParameter as e :
        assert True
      # print str(e)
    except Exception as e :
        assert False, "should have seen a BadParameter exception, not %s" % e

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4


from SAF.misc import saf_misc
from MobileApps.libs.ma_misc import ma_misc
from MobileApps.resources.const.android.const import TEST_DATA
import random
import pytest

def load_stack_info():
    stacks = saf_misc.load_json(ma_misc.get_abs_path(TEST_DATA.HPBRIDGE_TEST_STACK))
    if hasattr(pytest, "test_stack"):
        target_stack = pytest.test_stack
    else:
        target_stack = stacks["default_stack"]
    for stack in stacks["stacks"]:
        if stack["stack"] == target_stack:
            return stack

    raise KeyError("Failed to find stack for: %s " % target_stack)

def load_user_device():
    """
    This method used to load the test device and user info before AMS function integrated.
    :return:
    """
    devices = saf_misc.load_json(ma_misc.get_abs_path(TEST_DATA.HPBRIDGE_TEST_DEVICE))
    accounts = saf_misc.load_json(ma_misc.get_abs_path(TEST_DATA.HPBRIDGE_TEST_ACCOUNT))

    if hasattr(pytest, "test_mobile"):
        target_mobile = pytest.test_mobile
    else:
        target_mobile = devices["default_device"]

    for device in devices["devices"]:
        if device["device_name"] == target_mobile:
            test_device = device
            break
    if test_device is None:
        raise KeyError("Cannot find device for device name: %s" % devices["default_device"])

    for account in accounts["users"]:
        if account["nickname"] == test_device["wechat_user"]:
            test_user = account
            break
    if test_user is None:
        raise KeyError("Cannot find wechat user for user name: %s" % test_device["wechat_user"])

    return test_user, test_device

def load_printer(printer_name=None, stack=None, printer_type="Gen 1", status="normal", random_pick=False, api_used=False):
    """
    This method used to load the test printers which used for some special case, eg: using API to bind printer,
    then we need some device info in the HP bridge DB
    :param printer_name: the printer's name which to use
    :param stack: indicate the printer's stack
    :param printer_type: indicate the printer's type
    :param status: indicate the printer's status, if None, choose a printer which status should not be reset,
    offline and so on
    :param random_pick: if true. random to select a matchable printer, else, using the printer name or using default to select
    :param api_used: if true, default printer should be a printer with no index
    :return:
    """
    printer_info = saf_misc.load_json(ma_misc.get_abs_path(TEST_DATA.HPBRIDGE_TEST_PRINTER))
    if not stack:
        stack = load_stack_info()["stack"]
    if not printer_name and not random_pick:
        target_printer = printer_info[stack]["api_default"] if api_used else printer_info[stack]["ui_default"]
    else:
        target_printer = printer_name
    printers = []
    for printer in printer_info[stack]["printers"]:
        if printer["model"] == target_printer:
            return printer
        else:
            if target_printer:
                continue
            elif printer["type"] == printer_type and printer["status"] == status:
                printers.append(printer)

    if len(printers) != 0:
        index = random.randrange(0, len(printers))
        return printers[index]
    else:
        raise KeyError("Cannot find match printer in the json file with printer name: " + target_printer +
                       " stack: " + stack + " , status: " + status)

def get_reset_printer(stack=None):
    """
    This method used to load a reset printer from the printer.json file, only gen 1 printer has the reset status
    :param stack: the printer's stack
    :return:
    """
    return load_printer(stack, status="reset", printer_type="Gen 1", random_pick=True)

def get_offline_printer(stack=None, printer_type="Gen 2"):
    """
    This method used to load a offline printer from the printer.json file
    :param stack:  the printer's stack
    :param printer_type: indicate the printer's type
    :return:
    """
    return load_printer(stack, status="reset", printer_type=printer_type, random_pick=True)


#!/usr/bin/env python
# encoding: UTF-8

"""Exercise answer 16.8.1 for chapter 16."""

__author__ = 'Ibuki Suika'

from Tkinter import *
from ttk import *
from socket import *
from select import select
from threading import Thread
from time import ctime

class App(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.pack()
        self.master.title('Python版聊天程式')
        self.frm1 = Frame()
        self.frm1.pack(fill=BOTH)
        self.frm2 = Frame()
        self.frm2.pack(side=BOTTOM, fill=X)
        self.txt = Listbox(self.frm1, width=100, height=20)
        self.txt.pack(side=LEFT, fill=X)
        self.bar = Scrollbar(self.frm1)
        self.bar.pack(side=RIGHT, fill=Y)
        self.txt['yscrollcommand'] = self.bar.set
        self.bar['command'] = self.txt.yview
        self. lbl = Label(self.frm2, text='待发送：')
        self.lbl.pack(side=LEFT)
        self.content = StringVar()
        self.entry = Entry(self.frm2, width=80, textvariable=self.content)
        self.entry.pack(side=LEFT)
        self.btn = Button(self.frm2, text='发送', command=self.send_msg)
        self.btn.pack(side=LEFT)

    def send_msg(self):
        pass

class ServerApp(App):
    def __init__(self, host, port):
        App.__init__(self)
        self.server = socket(AF_INET, SOCK_STREAM)
        self.server.setblocking(False)
        self.server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
        self.server.bind((host, port))
        self.server.listen(1)
        self.inputs = [self.server]
        t = Thread(target=self.server_loop)
        t.setDaemon(True)
        t.start()

    def __del__(self):
        self.conn.close()

    def send_msg(self):
        s = self.content.get()
        self.txt.insert(END, '[%s] me:' % ctime())
        self.txt.insert(END, s)
        self.content.set('')
        if len(self.inputs) == 2:
            self.inputs[1].send(s)

    def server_loop(self):
        while True:
            readers, writers, exceptions = select(self.inputs, [], [])
            for reader in readers:
                if reader is self.server:
                    conn, addr = reader.accept()
                    conn.setblocking(False)
                    self.inputs.append(conn)
                else:
                    data = reader.recv(1024)
                    if data:
                        self.txt.insert(END, '[%s] stranger:' % ctime())
                        self.txt.insert(END, data)
                    else:
                        self.inputs.remove(reader)

if __name__ == '__main__':
    app = ServerApp('', 50007)
    app.mainloop()