import os
import os.path
import torch

IMG_EXTENSIONS = [
    '.jpg',
    '.JPG',
    '.jpeg',
    '.JPEG',
    '.png',
    '.PNG',
    '.ppm',
    '.PPM',
    '.bmp',
    '.BMP',
]

def is_image_file(filename):
    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)

def __absolute2relative(data):
    for i, item in enumerate(data):
        for j, path in enumerate(item):
            s_path = path.split('/')
            label = list(filter(lambda x:'monkaa' in x.lower() or 'driving' in x.lower() or 'flying' in x.lower(),s_path))[0]
            data[i][j] = '/'.join(s_path[s_path.index(label):])
    return data

def __relative2absolute(data, prefix):
    for i, item in enumerate(data):
        for j, path in enumerate(item):
            data[i][j] = os.path.join(prefix, path)
    return data

def dataloader(filepath):
    store_path = './flying3d_relative_pathlist_tuple_except_unused_files.pth'
    if os.path.exists(store_path):
        print('Loading dataset from {}'.format(store_path))
        store_data = torch.load(store_path)['data']
        return __relative2absolute(store_data, filepath)

    import sys
    print('The dataset is not flyingthings3D that I need.', file=sys.stderr)
    sys.exit(-1)

    classes = [
        d for d in os.listdir(filepath)
        if os.path.isdir(os.path.join(filepath, d))
    ]
    image = [img for img in classes if img.find('frames_cleanpass') > -1]
    disp = [dsp for dsp in classes if dsp.find('disparity') > -1]

    monkaa_path = filepath + [x for x in image if 'monkaa' in x][0]
    monkaa_disp = filepath + [x for x in disp if 'monkaa' in x][0]

    monkaa_dir = os.listdir(monkaa_path)

    all_left_img = []
    all_right_img = []
    all_left_disp = []
    test_left_img = []
    test_right_img = []
    test_left_disp = []

    for dd in monkaa_dir:
        for im in os.listdir(monkaa_path + '/' + dd + '/left/'):
            if is_image_file(monkaa_path + '/' + dd + '/left/' + im):
                all_left_img.append(monkaa_path + '/' + dd + '/left/' + im)
                all_left_disp.append(monkaa_disp + '/' + dd + '/left/' +
                                     im.split(".")[0] + '.pfm')

        for im in os.listdir(monkaa_path + '/' + dd + '/right/'):
            if is_image_file(monkaa_path + '/' + dd + '/right/' + im):
                all_right_img.append(monkaa_path + '/' + dd + '/right/' + im)

    flying_path = filepath + [x for x in image if 'flyingthings3d' in x][0]
    flying_disp = filepath + [x for x in disp if 'flyingthings3d' in x][0]
    flying_dir = flying_path + '/TRAIN/'
    subdir = ['A', 'B', 'C']

    for ss in subdir:
        flying = os.listdir(flying_dir + ss)

        for ff in flying:
            imm_l = os.listdir(flying_dir + ss + '/' + ff + '/left/')
            for im in imm_l:
                if is_image_file(flying_dir + ss + '/' + ff + '/left/' + im):
                    all_left_img.append(flying_dir + ss + '/' + ff + '/left/' +
                                        im)

                all_left_disp.append(flying_disp + '/TRAIN/' + ss + '/' + ff +
                                     '/left/' + im.split(".")[0] + '.pfm')

                if is_image_file(flying_dir + ss + '/' + ff + '/right/' + im):
                    all_right_img.append(flying_dir + ss + '/' + ff +
                                         '/right/' + im)

    flying_dir = flying_path + '/TEST/'

    subdir = ['A', 'B', 'C']

    for ss in subdir:
        flying = os.listdir(flying_dir + ss)

        for ff in flying:
            imm_l = os.listdir(flying_dir + ss + '/' + ff + '/left/')
            for im in imm_l:
                if is_image_file(flying_dir + ss + '/' + ff + '/left/' + im):
                    test_left_img.append(flying_dir + ss + '/' + ff +
                                         '/left/' + im)

                test_left_disp.append(flying_disp + '/TEST/' + ss + '/' + ff +
                                      '/left/' + im.split(".")[0] + '.pfm')

                if is_image_file(flying_dir + ss + '/' + ff + '/right/' + im):
                    test_right_img.append(flying_dir + ss + '/' + ff +
                                          '/right/' + im)

    driving_dir = filepath + [x for x in image if 'driving' in x][0] + '/'
    driving_disp = filepath + [x for x in disp if 'driving' in x][0]

    subdir1 = ['15mm_focallength', '35mm_focallength']
    subdir2 = ['scene_backwards', 'scene_forwards']
    subdir3 = ['fast', 'slow']

    for i in subdir1:
        for j in subdir2:
            for k in subdir3:
                imm_l = os.listdir(driving_dir + i + '/' + j + '/' + k +
                                   '/left/')
                for im in imm_l:
                    if is_image_file(driving_dir + i + '/' + j + '/' + k +
                                     '/left/' + im):
                        all_left_img.append(driving_dir + i + '/' + j + '/' +
                                            k + '/left/' + im)
                    all_left_disp.append(driving_disp + '/' + i + '/' + j +
                                         '/' + k + '/left/' +
                                         im.split(".")[0] + '.pfm')

                    if is_image_file(driving_dir + i + '/' + j + '/' + k +
                                     '/right/' + im):
                        all_right_img.append(driving_dir + i + '/' + j + '/' +
                                             k + '/right/' + im)

    all_data = (all_left_img, all_right_img, all_left_disp, test_left_img,
                test_right_img, test_left_disp)
    import copy
    save_data = __absolute2relative(copy.deepcopy(all_data))
    store_data = {'data': save_data}
    torch.save(store_data, store_path)

    return all_data

if __name__ == '__main__':
    dataloader('/home/caitao/Dataset/sceneflow/')


import numpy as np
import pandas as pd
from sklearn.manifold import TSNE

csv_data = pd.read_csv('/Users/darmanm/workspace/digitaltwin-experiments/clustering/material_embeddings.csv')
embeddings_data = csv_data.iloc[3:]
metadata = csv_data.iloc[:3]
print(embeddings_data.shape)

print("TSNE Transformation")
tsne = TSNE(n_components=3, n_iter=250)
tsne_result = tsne.fit_transform(embeddings_data)

print("Concating data, writing to CSV")
pd.concat([metadata, tsne_result], axis=1).write_csv('/Users/darmanm/workspace/digitaltwin-experiments/clustering/material_embeddings_tsne.csv')


"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.

Add arbitrary yum repo, specified by URL of repo file, to a list of
repos which should be injected into built image by the inject_yum_repo
plugin.

This plugin has to run _BEFORE_ the inject_yum_repo plugin, which
actually places the repo file in the build environment.

Example configuration to add content of repo file at URL:

{
    "name": "add_yum_repo_by_url",
    "args": {
        "repourls": ["http://example.com/myrepo/myrepo.repo"]
    }
}

"""
from atomic_reactor.constants import YUM_REPOS_DIR
from atomic_reactor.plugin import PreBuildPlugin
from atomic_reactor.util import get_retrying_requests_session
from hashlib import md5
import os
import os.path

try:
    # py2
    from urlparse import unquote, urlsplit
    import ConfigParser as configparser
    # We import BytesIO as StringIO as configparser can't properly write
    from io import BytesIO, BytesIO as StringIO
except ImportError:
    # py3
    from urllib.parse import unquote, urlsplit
    import configparser
    from io import BytesIO, StringIO

class YumRepo(object):
    def __init__(self, repourl, dst_repos_dir=YUM_REPOS_DIR):
        self.repourl = repourl
        self.dst_repos_dir = dst_repos_dir
        self.content = None

    @property
    def filename(self):
        '''Returns the filename to be used for saving the repo file.

        The filename is derived from the repo url by injecting a suffix
        after the name and before the file extension. This suffix is a
        partial md5 checksum of the full repourl. This avoids multiple
        repos from being written to the same file.
        '''
        urlpath = unquote(urlsplit(self.repourl, allow_fragments=False).path)
        basename = os.path.basename(urlpath)
        suffix = '-' + md5(self.repourl.encode('utf-8')).hexdigest()[:5]
        final_name = suffix.join(os.path.splitext(basename))
        return final_name

    @property
    def dst_filename(self):
        return os.path.join(self.dst_repos_dir, self.filename)

    def fetch(self):
        session = get_retrying_requests_session()
        response = session.get(self.repourl)
        response.raise_for_status()
        self.content = response.content

    def is_valid(self):
        # Using BytesIO as configparser in 2.7 can't work with unicode
        # see http://bugs.python.org/issue11597
        with BytesIO(self.content) as buf:
            self.config = configparser.ConfigParser()
            try:
                # Try python3 method
                try:
                    self.config.read_string(self.content.decode('unicode_escape'))
                except AttributeError:
                    # Fallback to py2 method
                    self.config.readfp(buf)
            except configparser.Error:
                self.log.warn("Invalid repo file found: '%s'", self.content)
                return False
            else:
                return True

    def set_proxy_for_all_repos(self, proxy_name):
        for section in self.config.sections():
            self.config.set(section, 'proxy', proxy_name)

        with StringIO() as output:
            self.config.write(output)
            self.content = output.getvalue()

class AddYumRepoByUrlPlugin(PreBuildPlugin):
    key = "add_yum_repo_by_url"
    is_allowed_to_fail = False

    def __init__(self, tasker, workflow, repourls=None, inject_proxy=None):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        :param repourls: list of str, URLs to the repo files
        :param inject_proxy: set proxy server for this repo
        """
        # call parent constructor
        super(AddYumRepoByUrlPlugin, self).__init__(tasker, workflow)
        self.repourls = repourls or []
        self.inject_proxy = inject_proxy

    def run(self):
        """
        run the plugin
        """
        if self.repourls:
            for repourl in self.repourls:
                yumrepo = YumRepo(repourl)
                yumrepo.fetch()
                self.log.info("fetched repo from '%s'", yumrepo.repourl)
                if self.inject_proxy:
                    if yumrepo.is_valid():
                        yumrepo.set_proxy_for_all_repos(self.inject_proxy)
                self.workflow.files[yumrepo.dst_filename] = yumrepo.content
                self.log.debug("saving repo '%s', length %d", yumrepo.dst_filename,
                               len(yumrepo.content))


"""add light_switch_state to sensor_data

Revision ID: e361748b3d99
Revises: 40efd4b5ddee
Create Date: 2017-06-27 20:41:23.199028

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'e361748b3d99'
down_revision = '40efd4b5ddee'
branch_labels = None
depends_on = None

def upgrade():
    op.add_column(
        'sensor_data',
        sa.Column('light_switch_state', sa.Boolean())
    )

def downgrade():
    op.drop_column('sensor_data', 'light_switch_state')


import time
import csv
import datetime
import urllib.request#获取okcoin行情,并储存为CSV文件
import json
import socket
def httpget(url,params='btc_cny'):
    url='https://www.okcoin.cn/api/v1/'+url+'?'+params
    response=urllib.request.urlopen(url,timeout=3)
    d=response.read().decode('utf-8')
    data=json.loads(d)
    return data
def saveData():
    #headers=['dateTime','timeStamp','highPrice','newestPrice','lowPrice','vol']
    timeStamp=int(ticker['date'])
    timeArray=time.localtime(timeStamp)
    dateTime=time.strftime("%Y-%m-%d %H:%M:%S",timeArray)
    timeStamp=ticker['date']
    highPrice=ticker['ticker']['high']
    newestPrice=ticker['ticker']['last']
    lowPrice=ticker['ticker']['low']
    vol=ticker['ticker']['vol']
    rows=[(dateTime,timeStamp,highPrice,newestPrice,lowPrice,vol)]
    filedate=datetime.datetime.now().strftime('%Y-%m-%d')
    with open('%s.csv'%filedate,'a') as f:
        f_csv = csv.writer(f)
        f_csv.writerows(rows)
    print("the newest price is %s"%newestPrice)
    return
while True:
    try:
        ticker=httpget('ticker.do')
        saveData()
        print('数据获取成功')
        time.sleep(5)
    except urllib.error.URLError:
        print('数据获取失败,正在重新获取...urlError')
        time.sleep(3)
    except socket.timeout:
        print('数据获取成功，正在重新获取...socket.time.out')
        time.sleep(3)


import http.client
import json
import requests
import re
from application.logger import get_middleware_logger
import math
from database import database
import datetime  # datetime
import time

LOGGER = get_middleware_logger('jijin_http')
myheaders = {'Content-type': 'application/json',"content":"text/html; charset=UTF-8"}
url = "http://vip.stock.finance.sina.com.cn/fund_center/data/jsonp.php/IO.XSRV2.CallbackList['Gb3sH5uawH5WCUZ9']/NetValue_Service.getNetValueOpen?page=1&num=40&sort=nav_date&asc=0&ccode=&type2=2&type3="
size:int=40

def get_api(url):
    """
    :param inputdata: 单个样本的输入参数，是json格式的数据
    :return: 单个样本的探真查询变量结果数据
    """
    # 调用接口
    response = requests.get(url=url, headers=myheaders)
    response.encoding = "UTF-8"
    LOGGER.info(response.json()['weatherinfo'])
    res = response.json()
    # 接口有正确的数据才读入，否则为空
    return res

def get_fund_history(fund_id):
    url = "http://vip.stock.finance.sina.com.cn/fund_center/data/jsonp.php/IO.XSRV2.CallbackList['Gb3sH5uawH5WCUZ9']/NetValue_Service.getNetValueOpen?page="+str(page)+"&num="+str(size)+"&sort=nav_date&asc=0&ccode=&type2=2&type3="
    print("获取page="+str(fund_id))
    print(url)
    response = requests.post(url,   headers = myheaders)
    print(response.text)
    response=re.findall(r"\((.+?)\);",response.text)
    data=response[0]
    print(data)
    res = json.loads(data)['data']
    return res

def get_page_finance(page):
    url = "http://vip.stock.finance.sina.com.cn/fund_center/data/jsonp.php/IO.XSRV2.CallbackList['Gb3sH5uawH5WCUZ9']/NetValue_Service.getNetValueOpen?page="+str(page)+"&num="+str(size)+"&sort=nav_date&asc=0&ccode=&type2=2&type3="
    print("获取page="+str(page))
    print(url)
    # json_foo = json.dumps(inputdata)
    response = requests.post(url,   headers = myheaders)
    print(response.text)
    response=re.findall(r"\((.+?)\);",response.text)
    data=response[0]
    print(data)
    res = json.loads(data)['data']
    return res

def update_fund_date_nav(fund_info):
    sql=" REPLACE  INTO `tianqi`.`fund_date_nav`(`fund_id`, `fund_name`, `per_nav`, `total_nav`, `yesterday_nav`, `nav_rate`, `nav_a`, `nav_date`, `fund_manager`, `fund_type`, `fund_total_num`) " \
        "VALUES ('%(fund_id)s','%(fund_name)s', '%(per_nav)s', '%(total_nav)s', '%(yesterday_nav)s', '%(nav_rate)s', '%(nav_a)s', '%(nav_date)s', '%(fund_manager)s', '%(fund_type)s', '%(fund_total_num)s');"\
        % {'fund_id':fund_info['fund_id'],'fund_name':fund_info['fund_name'],'per_nav':fund_info['per_nav'],'total_nav':fund_info['total_nav']
           ,'yesterday_nav':fund_info['yesterday_nav'],'nav_rate':fund_info['nav_rate'],'nav_a':fund_info['nav_a'],'nav_date':fund_info['nav_date']
           ,'fund_manager':fund_info['fund_manager'],'fund_type':fund_info['fund_type'],'fund_total_num':fund_info['fund_total_num']}
    print("更新：" +sql)
    database.insert(sql)

def update_fund_history_nav(fund_history_info):
    sql=" REPLACE  INTO `tianqi`.`fund_history_nav`(`fund_id`, `fund_date`, `fund_nav`, `fund_total_nav`) " \
        "VALUES ('%(fund_id)s','%(fund_date)s', '%(fund_nav)s', '%(fund_total_nav)s');"\
        % {'fund_id':fund_history_info['fund_id'],'fund_date':fund_history_info['fund_date'],'fund_nav':fund_history_info['fund_nav'],'fund_total_nav':fund_history_info['fund_total_nav']}
    print("更新：" +sql)
    database.insert(sql)

def get_all_finance():
    num=int(get_all_finance_num())
    pages=math.ceil(num/size)+1
    # pages=3
    fund_info={}

    time.sleep(5)
    for i in range(1,pages):
        res=get_page_finance(i)
        print(len(res))
        time.sleep(2)
        for x in res:
            print(x)
            time.sleep(1)
            fund_id=x['symbol']
            fund_info['fund_id']=fund_id
            fund_info['fund_name'] = x['sname']
            fund_info['per_nav'] = x['per_nav']
            fund_info['total_nav'] = x['total_nav']
            fund_info['yesterday_nav'] = x['yesterday_nav']
            fund_info['nav_rate'] = x['nav_rate']
            fund_info['nav_a'] = x['nav_a']
            fund_info['nav_date'] = x['nav_date']
            fund_info['fund_manager'] = x['fund_manager']
            fund_info['fund_type'] = x['jjlx']
            fund_info['fund_total_num'] = x['jjzfe']
            update_fund_date_nav(fund_info)
            get_fund_history(fund_id)
        print("=======")

t = datetime.date.today()
delat=datetime.timedelta(days=15)
format = "%Y-%m-%d"
dateto=t.strftime(format)
datefrom=(t-delat).strftime(format)
def get_finance_history_num(fund_id):
    url = "https://stock.finance.sina.com.cn/fundInfo/api/openapi.php/CaihuiFundInfoService.getNav?callback=jQuery1112006034245173649988_1597501048435&symbol="\
          +str(fund_id)+"&datefrom="+datefrom+"&dateto="+dateto+"&page=1&_=1597501048446"
    response = requests.post(url, headers = myheaders)
    print(response.text)
    response=re.findall(r"\((.+?)\)",response.text)
    print(response[0])
    res = json.loads(response[0])['result']['data']
    print(res)
    return res['total_num']

def get_finance_history_page(fund_id,page):
    url = "https://stock.finance.sina.com.cn/fundInfo/api/openapi.php/CaihuiFundInfoService.getNav?callback=jQuery1112006034245173649988_1597501048435&symbol="\
          +str(fund_id)+"&datefrom="+datefrom+"&dateto="+dateto+"&page="+str(page)+"&_=1597501048446"
    response = requests.post(url, headers = myheaders)
    print(response.text)
    response=re.findall(r"\((.+?)\)",response.text)
    print(response[0])
    res = json.loads(response[0])['result']['data']
    print(res)
    return res['data']
def get_fund_history(fund_id):
    histroy_size=21
    num=int(get_finance_history_num(fund_id))
    pages=math.ceil(num/histroy_size)+1
    fund_history_info={}
    for i in range(1,pages):
        res=get_finance_history_page(fund_id,i)
        print(len(res))
        for x in res:
            print(x)
            fund_history_info['fund_id']=fund_id
            fund_history_info['fund_date'] = x['fbrq']
            fund_history_info['fund_nav'] = x['jjjz']
            fund_history_info['fund_total_nav'] = x['ljjz']
            update_fund_history_nav(fund_history_info)
        print("=======")

def get_all_finance_num():

    inputdata={
        "page": 1,
        "num": 40,
        "sort": "nav_date",
        "asc": 0,
        "type2": 2,
    }
    json_foo = json.dumps(inputdata)
    response = requests.post(url, data =json_foo, headers = myheaders)
    print(response.text)
    response=re.findall(r"\((.+?)\);",response.text)
    print(response[0])
    res = json.loads(response[0])
    print(res['total_num'])
    return res['total_num']

if __name__ == '__main__':
    FORMAT = "%(asctime)s %(thread)d %(message)s"
    # print(get_fund_history("001113"))
    get_all_finance()


# Documentation: https://scikit-learn.org/stable/modules/neighbors.html#nearest-neighbor-algorithms

import meshio
import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import Rbf

mesh = meshio.read("circle-2d-drag_1000.vtu")
#cells=mesh.cells_dict
pointData=mesh.point_data
Velocity=pointData['Velocity'][:,[0,1]] # extracting the x and y velocity
points=mesh.points
nodal_coordinates=points[:,[0,1]] # extracting x,y data only

#%% Cut out the portion of the data before the cylinder
# see computing_the_cutoff_index.py in the same directory

xlowerlim =  0.44088176352705416
xupperlim =  1.8737474949899802
ylowerlim =  0.06180904522613064
yupperlim =  0.3399497487437185

# boolean to data to be trimmed

bool_data=np.zeros((len(nodal_coordinates)),dtype=bool)
# make sure the data is within x limit
x_limit_bool=np.logical_and(nodal_coordinates[:,0]<xupperlim, nodal_coordinates[:,0]>xlowerlim)
# make sure the data is within y limit
y_limit_bool=np.logical_and(nodal_coordinates[:,1]<yupperlim, nodal_coordinates[:,1]>ylowerlim)
# Using logical and to make sure the data is within both the limits
trimmed_bool_array=np.logical_and(x_limit_bool,y_limit_bool)
nodal_coordinates=nodal_coordinates[trimmed_bool_array]
Velocity=Velocity[trimmed_bool_array]

# for i in range (0,len(nodal_coordinates)):
#     bool_data[i]= ((nodal_coordinates[i,0]<xupperlim and nodal_coordinates[i,0]>xlowerlim) and (nodal_coordinates[i,1]<yupperlim and nodal_coordinates[i,1]>ylowerlim))

# nodal_coordinates[(nodal_coordinates[:,0]<xupperlim & nodal_coordinates[:,0]>xlowerlim) & (nodal_coordinates[:,1]<yupperlim & nodal_coordinates[:,1]>ylowerlim)]

# temp2=nodal_coordinates[bool_data]

#%% Point from structured grid

#Creating structured grid on trimmed mesh

# # Boundaries of the structured grid
# xlowerlim=(min(nodal_coordinates=[:,0]))
# xupperlim=(max(nodal_coordinates=[:,0]))

# ylowerlim=(min(nodal_coordinates=[:,1]))
# yupperlim=(max(nodal_coordinates=[:,1]))

# Number of nodes in the grid
num_x=500
num_y=200

x=np.linspace(xlowerlim,xupperlim,num_x)
y=np.linspace(ylowerlim,yupperlim,num_y)

x_meshgrid,y_meshgrid=np.meshgrid(x,y)

#%%
# RBF interpolation and prediction
def RBF_interpolant(x,y,values):
    return Rbf(x,y,values) # all these arrays should have same size

def RBF_prediction(RBF_model,x_meshgrid,y_meshgrid):
    return RBF_model(x_meshgrid,y_meshgrid)

RBF_model=RBF_interpolant(nodal_coordinates[:,0],nodal_coordinates[:,1],Velocity[:,0])
u_x=RBF_prediction(RBF_model,x_meshgrid,y_meshgrid)

plt.figure(2)
plt.pcolor(x_meshgrid,y_meshgrid,u_x,cmap='viridis')#,vmin=0, vmax=4)
plt.gca().set_aspect('equal', adjustable='box')
plt.colorbar(orientation='horizontal')

# It works really well

#%% Sample data points
'''
Credit of this part of code goes to:
Rudy SH, Brunton SL, Proctor JL, Kutz JN. Data-driven discovery of partial differential equations. Science Advances. 2017 Apr 1;3(4):e1602614.
Now randomly sample some points to use.
'''
print('Picking up random points from the data')
# Sample a collection of data points, stay away from edges so I can use interpolation.
# The coordinates is chosen randomly from the sliced data. That is why random.choice is used here
np.random.seed(0) # generates same set of random numbers each time
num_xy = 5000 # number of x and y points to be taken randomly in the collection
num_t = 60  # number of timesteps to be taken in the collection
num_points = num_xy * num_t  # total number of points in the collection
# Trimming the data that involves the cylinder
boundary = 5 # Selected points in y axis [boundary:m-boundary]
boundary_x = 11 # Selected points in x axis [boundary_x:n-boundary_x]
points = {}  # dictionary to store the collection
count = 0

3. nearest neighbour in the vtu file/nearest_neightbour.py


#! /usr/bin/env python
import logging
log = None
handler = None
LOG_PATH = '../data/log/'
#---------------------------------------
def init(filename):
	global log, handler
	if log and handler : return "Error: log is already initialized. Try log.close() first."
	if type(filename) != str or filename == '' : return "Error: wrong filename"
	log = logging.getLogger();
	log.setLevel(logging.INFO)
	filename = LOG_PATH + str(filename)
	handler = logging.FileHandler(filename)
	handler.setLevel(logging.INFO)
	formatter = logging.Formatter(
		fmt='%(asctime)s %(levelname)s: %(message)s',
		datefmt='%m-%d %H:%M'
		)
	handler.setFormatter(formatter)
	log.addHandler(handler)
	return
#------------------------------------
def msg(message):
	if not message : return "Error: empty message"
	if not log or not handler : return "Error: log is not initialized"
	log.info(message);
	return
#------------------------------------
def close():
	global log,handler
	if not log or not handler : return "Error: log is already closed."
	log.removeHandler(handler)
	log, handler = None, None
	return
#----------------------------------------


# -*- coding: utf-8 -*-
# :Project:   metapensiero.tool.bump_version -- Simple tool to bump a version number
# :Created:   dom  9 ago 2015, 14.28.40, CEST
# :Author:    Lele Gaifax <lele@metapensiero.it>
# :License:   GNU General Public License version 3 or later
# :Copyright: Copyright (C) 2015, 2017 Lele Gaifax
#

from io import open
import os

from setuptools import setup, find_packages

here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
    README = f.read()
with open(os.path.join(here, 'CHANGES.rst'), encoding='utf-8') as f:
    CHANGES = f.read()
with open(os.path.join(here, 'version.txt'), encoding='utf-8') as f:
    VERSION = f.read().strip()

setup(
    name="metapensiero.tool.bump_version",
    version=VERSION,
    url="https://bitbucket.org/lele/metapensiero.tool.bump_version",

    description="A simple tool to bump version number of a Python package",
    long_description=README + u'\n\n' + CHANGES,

    author="Lele Gaifax",
    author_email="lele@metapensiero.it",

    license="GPLv3+",
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        "Programming Language :: Python",
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.4",
        ],
    keywords='',

    packages=find_packages('src'),
    package_dir={'': 'src'},
    namespace_packages=['metapensiero', 'metapensiero.tool'],

    install_requires=['setuptools', 'Versio'],
    extras_require={
        'dev': [
            'metapensiero.tool.bump_version',
            'readme_renderer',
        ]
    },

    entry_points="""\
    [console_scripts]
    bump_version = metapensiero.tool.bump_version:main
    """,
)


from reportlab.lib.pagesizes import A4
from reportlab.lib.units import cm

W,H = A4

#margenes
m_top = H-2*cm #margen top
m_right = W-2*cm #margen
m_left = cm
m_bottom = cm

y_pos = m_top #posicion actual del cursor
y_rest = m_top - m_bottom #alto disponible para dibujo
x_rest = m_right - m_left
y_anchor = 0 #separacion vertical

#tamanios de texto
H1 = 34
H2 = 23
H3 = 18
P = 12


from marshmallow import fields, Schema

import models

class ItemSchema(Schema):
    id = fields.Int(dump_only=True)
    name = fields.Str()
    description = fields.Str()
    complete = fields.Boolean()

def init_app(app):
    app.add_url_rule('/v1/items', handler=get_items, method='GET')
    app.add_url_rule('/v1/items', handler=create_item, method='POST')
    app.add_url_rule('/v1/items/<int:id>', handler=update_item, method='PUT')
    app.add_url_rule('/v1/items/<int:id>', handler=delete_item, method='DELETE')
    app.add_url_rule('/v1/items/<int:id>', handler=get_item_by_id, method='GET')

def get_items(session, **kwargs):
    query_params = kwargs.get('querystring', {})
    items = models.get_items(session, incomplete_only=query_params.get('exclude_complete', True))

    items = ItemSchema(many=True).dump(items)

    return {'items': items.data}

def create_item(session, **kwargs):
    body = kwargs.get('body', {})
    item_kwargs, errors = ItemSchema().load(body)

    if errors:
        raise Exception

    item = models.create_item(session, **item_kwargs)
    session.commit()

    item = ItemSchema().dump(item)

    return {'item': item.data}

def get_item_by_id(session, id, **kwargs):
    item = models.get_items(session, id=id, incomplete_only=False).first()
    if not item:
        raise Exception

    item = ItemSchema().dump(item)

    return {'item': item.data}

def update_item(session, id, **kwargs):
    body = kwargs.get('body', {})
    item_kwargs, errors = ItemSchema().load(body)
    if errors:
        raise Exception

    item = models.update_item(session, id, **item_kwargs)
    if not item:
        raise Exception

    session.commit()

    item = ItemSchema().dump(item)

    return {'item': item.data}

def delete_item(session, id, **kwargs):
    item = models.delete_item(session, id)
    if not item:
        raise Exception

    session.commit()

    item = ItemSchema().dump(item)

    return {'item': item.data}


import json
import datetime
import math

from airflow import DAG
from airflow.models import Variable
from airflow.contrib.kubernetes import secret
from airflow.contrib.operators import kubernetes_pod_operator
from airflow.operators.python_operator import PythonOperator
from helper_dag import generate_splitted_batches_for_audio_analysis

audio_analysis_config = json.loads(Variable.get("audio_analysis_config"))
source_path = Variable.get("sourcepathforaudioanalysis")
destination_path = Variable.get("destinationpathforaudioanalysis")
bucket_name = Variable.get("bucket")
env_name = Variable.get("env")
composer_namespace = Variable.get("composer_namespace")
project = Variable.get("project")

resource_limits = json.loads(Variable.get("audio_analysis_resource_limits"))

YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)
LANGUAGE_CONSTANT = "{language}"

secret_file = secret.Secret(
    deploy_type="volume",
    deploy_target="/tmp/secrets/google",
    secret="gc-storage-rw-key",
    key="key.json",
)

def interpolate_language_paths(path, language):
    path_set = path.replace(LANGUAGE_CONSTANT, language)
    return path_set

def create_dag(dag_id, dag_number, default_args, args, max_records_threshold_per_pod):
    dag = DAG(
        dag_id,
        schedule_interval=datetime.timedelta(days=1),
        default_args=default_args,
        start_date=YESTERDAY,
    )

    with dag:
        audio_format = args.get("audio_format")
        language = args.get("language")
        parallelism = args.get("parallelism")
        source = args.get("source")
        print(args)
        print(f"Language for source is {language}")
        source_path_set = interpolate_language_paths(source_path, language)
        destination_path_set = interpolate_language_paths(destination_path, language)
        generate_batch_files = PythonOperator(
            task_id=dag_id + "_generate_batches",
            python_callable=generate_splitted_batches_for_audio_analysis,
            op_kwargs={
                "source": source,
                "source_path": source_path_set,
                "destination_path": destination_path_set,
                "max_records_threshold_per_pod": max_records_threshold_per_pod,
                "audio_format": audio_format,
                "bucket_name": bucket_name,
            },
            dag_number=dag_number,
        )

        generate_batch_files

        data_audio_analysis_task = kubernetes_pod_operator.KubernetesPodOperator(
            task_id=f"data-audio-analysis-{source}",
            name="data-audio-analysis",
            cmds=[
                "python",
                "invocation_script.py",
                "-b",
                bucket_name,
                "-a",
                "audio_analysis",
                "-rc",
                f"data/audiotospeech/config/config.yaml",
                "-as",
                source,
                "-l",
                language,
            ],
            namespace=composer_namespace,
            startup_timeout_seconds=300,
            secrets=[secret_file],
            image=f"us.gcr.io/{project}/ekstep_data_pipelines:{env_name}_1.0.0",
            image_pull_policy="Always",
            resources=resource_limits,
        )

        batch_file_path_dict = json.loads(Variable.get("embedding_batch_file_list"))
        list_of_batch_files = batch_file_path_dict[source]
        if len(list_of_batch_files) > 0:
            total_phases = math.ceil(len(list_of_batch_files) / parallelism)
            task_dict = {}
            for phase in range(0, total_phases):
                for pod in range(1, parallelism + 1):
                    if len(list_of_batch_files) > 0:
                        batch_file = list_of_batch_files.pop()
                        task_dict[
                            f"create_embedding_task_{pod}_{phase}"] = kubernetes_pod_operator.KubernetesPodOperator(
                            task_id=source + "_create_embedding_" + str(pod) + '_' + str(phase),
                            name="create-embedding",
                            cmds=[
                                "python",
                                "invocation_script.py",
                                "-b",
                                bucket_name,
                                "-a",
                                "audio_embedding",
                                "-rc",
                                f"data/audiotospeech/config/config.yaml",
                                "-fp",
                                batch_file,
                                "-as",
                                source,
                                "-l",
                                language,
                            ],
                            namespace=composer_namespace,
                            startup_timeout_seconds=300,
                            secrets=[secret_file],
                            image=f"us.gcr.io/{project}/ekstep_data_pipelines:{env_name}_1.0.0",
                            image_pull_policy="Always",
                            resources=resource_limits,
                        )
                        if phase == 0:
                            generate_batch_files >> task_dict[
                                f"create_embedding_task_{pod}_{phase}"] >> data_audio_analysis_task
                        # elif phase == total_phases - 1:
                        #     task_dict[f"create_embedding_task_{pod}_{phase - 1}"] >> task_dict[
                        #         f"create_embedding_task_{pod}_{phase}"] >> data_audio_analysis_task
                        else:
                            task_dict[f"create_embedding_task_{pod}_{phase - 1}"] >> task_dict[
                                f"create_embedding_task_{pod}_{phase}"] >> data_audio_analysis_task
            # if phase == total_phases - 1:
            #     task_dict[f"create_embedding_task_{last_pod_no}_{phase}"] >> data_audio_analysis_task
        else:
            generate_batch_files >> data_audio_analysis_task

    return dag

for source in audio_analysis_config.keys():
    source_info = audio_analysis_config.get(source)

    max_records_threshold_per_pod = source_info.get("batch_size")
    parallelism = source_info.get("parallelism", 5)
    audio_format = source_info.get("format")
    language = source_info.get("language").lower()

    dag_id = source + '_' + 'audio_embedding_analysis'

    dag_args = {
        "email": ["soujyo.sen@thoughtworks.com"],
    }

    args = {
        "audio_format": audio_format,
        "parallelism": parallelism,
        "language": language,
        "source": source
    }

    # schedule = '@daily'

    dag_number = dag_id + str(max_records_threshold_per_pod)

    globals()[dag_id] = create_dag(dag_id, dag_number, dag_args, args, max_records_threshold_per_pod)


# -*- coding:utf-8 -*-
import os
from cloud import app
from flask_wtf import FlaskForm
from wtforms import SubmitField
from flask import request,flash,render_template,redirect
from flask_wtf.file import FileField, FileRequired, FileAllowed
from flask_uploads import UploadSet, configure_uploads, IMAGES,DATA ,\
 patch_request_class

FLASKR_SETTINGS=os.path.join(app.root_path, 'setting.cfg')
app.config.from_pyfile(FLASKR_SETTINGS, silent=True)
pathReal=app.config['UPLOADED_DATA_DEST']

Data_list = os.listdir(pathReal) #文件位置
Data_file = UploadSet('data', DATA ) #指定文件类型

configure_uploads(app, Data_file)
patch_request_class(app) #文件大小

class UploadForm(FlaskForm):
    file = FileField(validators=[FileAllowed(Data_file, u'Data_file Only!'), \
            FileRequired(u'Choose a file!')])  #表单名
    submit = SubmitField(u'Upload')

@app.route('/del')
def del_file():
    Data_file_path=[Data_file.path(x) for x in Data_list]
    [os.remove(x) for x in Data_file_path]
    return render_template('del.html',Data_file_path=Data_file_path)

@app.route('/ds_upload', methods=['GET', 'POST'])
def ds_upload():
    form = UploadForm()
    file_urls= [Data_file.url(x)  for x in os.listdir(app.config['UPLOADED_DATA_DEST'])]
    if form.validate_on_submit():
        filename = Data_file.save(form.file.data) #安全文件名
        if fn == filename :
            os.remove(Data_file.path(filename))
        fn = filename
        file_urls.append(Data_file.url(filename))
        flash(file_urls)
    else:
        file_url = None
        flash("wait")
    return render_template('ds_upload.html', form=form, file_urls=file_urls)

@app.route('/uploads', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        al=True
        for f in request.files.getlist('file'):
            ls = os.listdir(pathReal) #去重
            if f.filename not in ls:
                f.save(os.path.join(pathReal,f.filename))
                ls.append(f.filename)
            # Data_list.append(os.path.join(pathReal, f))
    flash("(This is just a demo. Selected files are not actually uploaded to analyse)")
    return render_template('upload.html')

@app.route('/')
def index():
    return render_template('index.html')


# coding: utf-8

# In[1]:

import pandas as pd
import os
import matplotlib.pyplot as plt
import statsmodels.api as sm
os.chdir('d:/')
import numpy as np

# In[2]:

df=pd.read_csv('wcs crude cadaud.csv',encoding='utf-8')

# In[3]:

df.set_index('date',inplace=True)

# In[4]:

df.index=pd.to_datetime(df.index,format='%m/%d/%Y')

# In[5]:

df=df.reindex(columns=
['cny',
 'gbp',
 'usd',
 'eur',
 'krw',
 'mxn',
 'gas',
 'wcs',
 'edmonton',
 'wti',
 'gold',
 'jpy',
 'cad'])

# In[6]:

var=locals()

for i in df.columns:
    if i!='cad':
            x=sm.add_constant(df[i])
            y=df['cad']
            m=sm.OLS(y,x).fit()
            var[str(i)]=m.rsquared

ax=plt.figure(figsize=(10,5)).add_subplot(111)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)

width=0.7
colorlist=['#9499a6','#9499a6','#9499a6','#9499a6',
           '#9499a6','#9499a6','#9499a6','#582a20',
           '#be7052','#f2c083','#9499a6','#9499a6']

temp=list(df.columns)
for i in temp:
    if i!='cad':
        plt.bar(temp.index(i)+width,
            var[str(i)],width=width,label=i,
               color=colorlist[temp.index(i)])

plt.title('Regressions on Loonie')
plt.ylabel('R Squared\n')
plt.xlabel('\nRegressors')
plt.xticks(np.arange(len(temp))+width,
           ['Yuan', 'Sterling', 'Dollar', 'Euro', 'KRW',
             'MXN', 'Gas', 'WCS', 'Edmonton',
             'WTI', 'Gold', 'Yen'],fontsize=10)
plt.show()

# In[7]:

ax=plt.figure(figsize=(10,5)).add_subplot(111)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)

(df['cny']/df['cny'].iloc[0]).plot(c='#283670',label='Yuan')
(df['gbp']/df['gbp'].iloc[0]).plot(c='#fe4d32',label='Sterling')
(df['cad']/df['cad'].iloc[0]).plot(c='#484a25',label='Loonie')
plt.legend(loc=0)
plt.xlabel('Date')
plt.ylabel('Normalized Value by 100')
plt.title('Loonie vs Yuan vs Sterling')
plt.show()

# In[8]:

ax=plt.figure(figsize=(10,5)).add_subplot(111)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)

(df['wti']/df['wti'].iloc[0]).plot(c='#2a78b2',label='WTI',alpha=0.5)
(df['wcs']/df['wcs'].iloc[0]).plot(c='#7b68ee',label='WCS',alpha=0.5)
(df['edmonton']/df['edmonton'].iloc[0]).plot(c='#110b3c',
                                             label='Edmonton',alpha=0.5)
(df['cad']/df['cad'].iloc[0]).plot(c='#cb8b8b',label='Loonie')
plt.legend(loc=0)
plt.xlabel('Date')
plt.ylabel('Normalized Value by 100')
plt.title('Loonie vs Crude Oil Blends')
plt.show()


import keras
from keras.models import Sequential, Model, load_model
from keras.layers.core import Activation
from keras.layers import Dense, Dropout, Activation, Flatten, Input, Lambda
from keras.layers import Conv2D, MaxPooling2D, Conv1D, MaxPooling1D, LSTM, ConvLSTM2D, GRU, BatchNormalization, LocallyConnected2D, Permute
from keras.layers import Concatenate, Reshape, Softmax, Conv2DTranspose, Embedding, Multiply
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras import regularizers
from keras import backend as K
import keras.losses

import tensorflow as tf

import isolearn.keras as iso

import numpy as np

#DragoNN Saved Model definition

def load_saved_predictor(model_path, library_context=None) :

	saved_model = Sequential()

	saved_model.add(Conv1D(120, 5, activation='relu', input_shape=(145, 4), name='dragonn_conv1d_1_copy'))
	saved_model.add(BatchNormalization(name='dragonn_batchnorm_1_copy'))
	saved_model.add(Dropout(0.1))

	saved_model.add(Conv1D(120, 5, activation='relu', name='dragonn_conv1d_2_copy'))
	saved_model.add(BatchNormalization(name='dragonn_batchnorm_2_copy'))
	saved_model.add(Dropout(0.1))

	saved_model.add(Conv1D(120, 5, activation='relu', name='dragonn_conv1d_3_copy'))
	saved_model.add(BatchNormalization(name='dragonn_batchnorm_3_copy'))
	saved_model.add(Dropout(0.1))

	saved_model.add(Flatten())
	saved_model.add(Dense(12, activation='linear', name='dragonn_dense_1_copy'))

	saved_model.compile(
		loss= "mean_squared_error",
		optimizer=keras.optimizers.SGD(lr=0.1)
	)

	saved_model.load_weights(model_path)
	#print(saved_model.summary())

	def _initialize_predictor_weights(predictor_model, saved_model=saved_model) :
		#Load pre-trained model
		predictor_model.get_layer('dragonn_conv1d_1').set_weights(saved_model.get_layer('dragonn_conv1d_1_copy').get_weights())
		predictor_model.get_layer('dragonn_conv1d_1').trainable = False
		predictor_model.get_layer('dragonn_batchnorm_1').set_weights(saved_model.get_layer('dragonn_batchnorm_1_copy').get_weights())
		predictor_model.get_layer('dragonn_batchnorm_1').trainable = False

		predictor_model.get_layer('dragonn_conv1d_2').set_weights(saved_model.get_layer('dragonn_conv1d_2_copy').get_weights())
		predictor_model.get_layer('dragonn_conv1d_2').trainable = False
		predictor_model.get_layer('dragonn_batchnorm_2').set_weights(saved_model.get_layer('dragonn_batchnorm_2_copy').get_weights())
		predictor_model.get_layer('dragonn_batchnorm_2').trainable = False

		predictor_model.get_layer('dragonn_conv1d_3').set_weights(saved_model.get_layer('dragonn_conv1d_3_copy').get_weights())
		predictor_model.get_layer('dragonn_conv1d_3').trainable = False
		predictor_model.get_layer('dragonn_batchnorm_3').set_weights(saved_model.get_layer('dragonn_batchnorm_3_copy').get_weights())
		predictor_model.get_layer('dragonn_batchnorm_3').trainable = False

		predictor_model.get_layer('dragonn_dense_1').set_weights(saved_model.get_layer('dragonn_dense_1_copy').get_weights())
		predictor_model.get_layer('dragonn_dense_1').trainable = False

	def _initialize_predictor_weights_old(predictor_model, saved_model=saved_model, model_path=model_path) :
		#Load pre-trained model
		#print(saved_model.summary())
		predictor_model.load_weights(model_path, by_name=True)

	def _load_predictor_func(sequence_input) :
		#DragoNN parameters
		seq_length = 145
		seq_input_shape = (seq_length, 4, 1)
		n_tasks = 1

		#Define model layers
		permute_input = Lambda(lambda x: x[..., -1])

		conv_1 = Conv1D(120, 5, padding='valid', activation='relu', name='dragonn_conv1d_1')
		batchnorm_1 = BatchNormalization(axis=-1, name='dragonn_batchnorm_1')
		drop_1 = Dropout(0.1)

		conv_2 = Conv1D(120, 5, padding='valid', activation='relu', name='dragonn_conv1d_2')
		batchnorm_2 = BatchNormalization(axis=-1, name='dragonn_batchnorm_2')
		drop_2 = Dropout(0.1)

		conv_3 = Conv1D(120, 5, padding='valid', activation='relu', name='dragonn_conv1d_3')
		batchnorm_3 = BatchNormalization(axis=-1, name='dragonn_batchnorm_3')
		drop_3 = Dropout(0.1)

		flatten_3 = Flatten()
		final_dense = Dense(12, name='dragonn_dense_1')

		#Execute functional model definition
		permuted_input = permute_input(sequence_input)

		drop_1_out = drop_1(batchnorm_1(conv_1(permuted_input), training=False), training=False)
		drop_2_out = drop_2(batchnorm_2(conv_2(drop_1_out), training=False), training=False)
		drop_3_out = drop_3(batchnorm_3(conv_3(drop_2_out), training=False), training=False)

		final_dense_out = final_dense(flatten_3(drop_3_out))

		predictor_inputs = []
		predictor_outputs = [final_dense_out, drop_1_out, drop_2_out, drop_3_out]

		return predictor_inputs, predictor_outputs, _initialize_predictor_weights

	return _load_predictor_func


'''
Created on 27 janv. 2018

@author: Maxime
'''
from django.urls import path
from . import views

urlpatterns = [
    path('', views.connexion, name='connexion'),
    path('index/', views.index, name='index'),
    path('dispatch', views.dispatcher, name='dispatch'),
    path('listeHebergeurs/', views.listeHebergeurs, name='listeHebergeurs'),
    path('listeHebergeurs/<int:hebergeur_id>/', views.listeHebergeurs, name='listeHebergeurs'),
    path('hebergeurCreateOrUpdate/', views.hebergeurCreateOrUpdate, name='hebergeurCreateOrUpdate'),
    path('hebergeurCreateOrUpdate/<int:hebergeur_id>/', views.hebergeurCreateOrUpdate, name='hebergeurCreateOrUpdate'),
    path('modifyHebergeur/', views.modifyHebergeur, name='modifyHebergeur'),
    path('listeHebergeurs/delete/<int:pk>/', views.deleteHebergeur, name='deleteHebergeur'),
    path('listeJeunes/', views.listeJeunes, name='listeJeunes'),
    path('listeJeunes/<int:jeune_id>/', views.listeJeunes, name='listeJeunes'),
    path('jeuneCreateOrUpdate/', views.jeuneCreateOrUpdate, name='jeuneCreateOrUpdate'),
    path('jeuneCreateOrUpdate/<int:jeune_id>/', views.jeuneCreateOrUpdate, name='jeuneCreateOrUpdate'),
    path('modifyJeune/', views.modifyJeune, name='modifyJeune'),
    path('listeJeunes/delete/<int:pk>/', views.deleteJeune, name='deleteJeune'),
    path('deconnexion', views.deconnexion, name='deconnexion'),
]


import pandas as pd
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from sklearn.cluster import KMeans, DBSCAN
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import silhouette_score, silhouette_samples
import numpy as np
import matplotlib.cm as cm
from sklearn.decomposition import PCA
import seaborn as sns

def textPreprocessing(df):
    stop = stopwords.words('english')
    df['text'] = df['text'].str.replace('[^\w\s]', '')
    df['text'] = df['text'].apply(lambda x: " ".join(x for x in x.split() if x not in stop))
    df['text'] = df['text'].apply(lambda x: " ".join(x.lower() for x in x.split()))
    stemmer = PorterStemmer()

    df['text'] = df['text'].apply(lambda x: " ".join([stemmer.stem(word) for word in x.split()]))
    df['text'] = df['text'].apply(lambda x: " ".join(x for x in x.split() if x not in stop))

    return df

def TFIDFtransformer(df):
    tfidf = TfidfVectorizer(max_features=100, analyzer='word', ngram_range=(1, 3), stop_words='english')
    tfidfDF = pd.DataFrame(tfidf.fit_transform(df['text']).toarray())
    cols = tfidf.get_feature_names()
    return tfidfDF, cols

# elbow rule plot
def optimalK_SSE(data):
    sum_of_squared_distances = []
    K = range(2, 11)
    for k in K:
        km = KMeans(n_clusters=k)
        km = km.fit(data)
        sum_of_squared_distances.append(km.inertia_)

    plt.plot(K, sum_of_squared_distances, 'bx-')
    plt.xlabel('k')
    plt.ylabel('Sum_of_squared_distances')
    plt.title('Elbow Method For Optimal k')
    plt.show()

def optimalK_Silouhette(data):
    silhouette_scores = []
    K = range(2, 11)
    for k in K:
        km = KMeans(n_clusters=k)
        cluster_labels = km.fit_predict(data)
        silhouette_avg = silhouette_score(data, cluster_labels)
        silhouette_scores.append(silhouette_avg)

    plt.plot(K, silhouette_scores, 'bx-')
    plt.xlabel('k')
    plt.ylabel('Silhouette Scores')
    plt.title('Silhouette For Optimal k')
    plt.show()

def plotSilouhette(X):
    range_n_clusters = range(2, 11)

    for n_clusters in range_n_clusters:
        # Create a subplot with 1 row and 2 columns
        fig, (ax1, ax2) = plt.subplots(1, 2)
        fig.set_size_inches(18, 7)

        # The 1st subplot is the silhouette plot
        # The silhouette coefficient can range from -1, 1 but in this example all
        # lie within [-0.1, 1]
        ax1.set_xlim([-0.1, 1])
        # The (n_clusters+1)*10 is for inserting blank space between silhouette
        # plots of individual clusters, to demarcate them clearly.
        ax1.set_ylim([0, len(X) + (n_clusters + 1) * 10])

        # Initialize the clusterer with n_clusters value and a random generator
        # seed of 10 for reproducibility.
        clusterer = KMeans(n_clusters=n_clusters, random_state=10)
        cluster_labels = clusterer.fit_predict(X)

        # The silhouette_score gives the average value for all the samples.
        # This gives a perspective into the density and separation of the formed
        # clusters
        silhouette_avg = silhouette_score(X, cluster_labels)
        print("For n_clusters =", n_clusters,
              "The average silhouette_score is :", silhouette_avg)

        # Compute the silhouette scores for each sample
        sample_silhouette_values = silhouette_samples(X, cluster_labels)

        y_lower = 10
        for i in range(n_clusters):
            # Aggregate the silhouette scores for samples belonging to
            # cluster i, and sort them
            ith_cluster_silhouette_values = \
                sample_silhouette_values[cluster_labels == i]

            ith_cluster_silhouette_values.sort()

            size_cluster_i = ith_cluster_silhouette_values.shape[0]
            y_upper = y_lower + size_cluster_i

            color = cm.nipy_spectral(float(i) / n_clusters)
            ax1.fill_betweenx(np.arange(y_lower, y_upper),
                              0, ith_cluster_silhouette_values,
                              facecolor=color, edgecolor=color, alpha=0.7)

            # Label the silhouette plots with their cluster numbers at the middle
            ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))

            # Compute the new y_lower for next plot
            y_lower = y_upper + 10  # 10 for the 0 samples

        ax1.set_title("The silhouette plot for the various clusters.")
        ax1.set_xlabel("The silhouette coefficient values")
        ax1.set_ylabel("Cluster label")

        # The vertical line for average silhouette score of all the values
        ax1.axvline(x=silhouette_avg, color="red", linestyle="--")

        ax1.set_yticks([])  # Clear the yaxis labels / ticks
        ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])

        # 2nd Plot showing the actual clusters formed
        colors = cm.nipy_spectral(cluster_labels.astype(float) / n_clusters)
        ax2.scatter(X.iloc[:, 0], X.iloc[:, 1], marker='.', s=30, lw=0, alpha=0.7,
                    c=colors, edgecolor='k')

        # Labeling the clusters
        centers = clusterer.cluster_centers_
        # Draw white circles at cluster centers
        ax2.scatter(centers[:, 0], centers[:, 1], marker='o',
                    c="white", alpha=1, s=200, edgecolor='k')

        for i, c in enumerate(centers):
            ax2.scatter(c[0], c[1], marker='$%d$' % i, alpha=1,
                        s=50, edgecolor='k')

        ax2.set_title("The visualization of the clustered data.")
        ax2.set_xlabel("Feature space for the 1st feature")
        ax2.set_ylabel("Feature space for the 2nd feature")

        # plt.suptitle(("Silhouette analysis for KMeans clustering on sample data "
        #               "with n_clusters = %d" % n_clusters),
        #              fontsize=14, fontweight='bold')

    plt.show()

def intepretationWithVariance():

    print()

if __name__ == '__main__':

    # load reviews csv
    # df = pd.read_csv('/home/andreas/Documents/Notebooks/TripAdvisor/reviews.csv')
    # print(df.head())
    # print(df.dtypes)
    #
    # # keep ony text
    # df = df[['username', 'text']]
    #
    # # group by username & merge reviews text to a unified corpus
    # df = df.groupby('username').agg({
    #     'text': lambda x: ' '.join(x)
    # }).reset_index()
    #
    # # Text Pre-Processing
    # df = textPreprocessing(df)
    #
    # # Text Vectorization using TFIDF
    # df, cols = TFIDFtransformer(df)
    # df.columns = cols
    # print(df.head())
    #
    # df.to_csv('tfidf.csv', index=False)
    df = pd.read_csv('tfidf.csv')

    # print('before outliers:', len(df))
    # remove outliers
    # from scipy import stats
    # df = df[(np.abs(stats.zscore(df)) < 3).all(axis=1)]
    # print('after outliers:', len(df))

    pca = PCA(n_components=2)
    pcadf = pd.DataFrame(pca.fit_transform(df))

    print(pca.explained_variance_ratio_)

    # df.to_csv('pcaDF.csv', index=False)

    # pcadf = pd.read_csv('pcaDF.csv')

    # Clustering Evaluation
    optimalK_SSE(pcadf)
    optimalK_Silouhette(pcadf)
    # plotSilouhette(pcadf)

    # dbscan = DBSCAN(eps=1, min_samples=2).fit(df)
    # print('DBSCAN: {}'.format(silhouette_score(df, dbscan.labels_,
    #                                            metric='cosine')))

    km = KMeans(n_clusters=3).fit(pcadf)
    # cluster_labels = km.fit_predict(df)

    from sklearn.preprocessing import MinMaxScaler

    scaler = MinMaxScaler()
    df_scaled = pd.DataFrame(scaler.fit_transform(df))
    df_scaled['cluster_id'] = km.labels_
    df_mean = df_scaled.groupby('cluster_id').mean()
    # df_mean.columns  = df.columns

    results = pd.DataFrame(columns=['Variable', 'Var'])
    for column in df_mean.columns:
        print(column)
        results.loc[len(results), :] = [column, np.var(df_mean[column])]
    selected_columns = list(results.sort_values(
        'Var', ascending=False,
    ).head(15).Variable.values) + ['cluster_id']
    tidy = df_scaled[selected_columns].melt(id_vars='cluster_id')
    tidy['variable'] = tidy['variable'].apply(lambda x: df.columns[x])
    # clrs = ['grey' if (x < max(tidy['value'])) else 'red' for x in tidy['value']]
    sns.barplot(x='cluster_id', y='value', hue='variable', data=tidy)
    plt.legend(bbox_to_anchor=(1.01, 1),
               borderaxespad=0)
    plt.title('Interpretation with feature variance')
    plt.show()

    # for i in selected_columns:
    #     print(str(i) + ': ' + str(df.columns[i]))

    from sklearn.ensemble import RandomForestClassifier

    X, y = df_scaled.iloc[:, :-1], df_scaled.iloc[:, -1]
    clf = RandomForestClassifier(n_estimators=100).fit(X, y)
    data = np.array([clf.feature_importances_, X.columns]).T
    columns = list(pd.DataFrame(data, columns=['Importance', 'Feature'])
                   .sort_values("Importance", ascending=False)
                   .head(15).Feature.values)
    tidy = df_scaled[columns + ['cluster_id']].melt(id_vars='cluster_id')
    tidy['variable'] = tidy['variable'].apply(lambda x: df.columns[x])
    sns.barplot(x='cluster_id', y='value', hue='variable', data=tidy)
    plt.legend(bbox_to_anchor=(1.01, 1),
               borderaxespad=0)
    plt.title('Interpretation with feature importance')
    plt.show()


from MockData.names_list import last_names, female_names, male_names
import random
import pprint
import json
import MockData.config as config
import pickle
import datetime
import unicodedata
import string
import random
from MockData.config import source_path, data_path

pp = pprint.PrettyPrinter(indent=4)

GENDER_BIAS = 53
last_name = {'last_name' : None, 'case' : None }
first_name = {'given_name' : None, 'case' : None, 'gender' : None, 'ordinal' : 0  }
full_name = { 'first_to_last' : None, 'last_then_first' : None,  'gender' : None,
              'given_names' : [], 'surname' : None }

stateNamexRef = {'Mississippi': 'MS', 'Iowa': 'IA', 'Oklahoma': 'OK',
    'Wyoming': 'WY', 'Minnesota': 'MN', 'Illinois': 'IL', 'Arkansas': 'AR',
    'New Mexico': 'NM', 'Indiana': 'IN', 'Maryland': 'MD', 'Louisiana': 'LA',
    'Texas': 'TX', 'Arizona': 'AZ', 'Wisconsin': 'WI', 'Michigan': 'MI',
    'Kansas': 'KS', 'Utah': 'UT', 'Virginia': 'VA', 'Oregon': 'OR',
    'Connecticut': 'CT', 'Montana': 'MT', 'California': 'CA', 'Massachusetts': 'MA',
    'West Virginia': 'WV', 'Delaware': 'DE', 'New Hampshire': 'NH', 'Vermont': 'VT',
    'Georgia': 'GA', 'North Dakota': 'ND', 'Hawaii': 'HI', 'Pennsylvania': 'PA',
    'Puerto Rico': 'PR', 'Florida': 'FL', 'Alaska': 'AK', 'Kentucky': 'KY',
    'Tennessee': 'TN', 'South Carolina': 'SC', 'Nebraska': 'NE', 'Missouri': 'MO',
    'Ohio': 'OH', 'Alabama': 'AL', 'Rhode Island': 'RI', 'South Dakota': 'SD',
    'Colorado': 'CO', 'Idaho': 'ID', 'New Jersey': 'NJ', 'Washington': 'WA',
    'North Carolina': 'NC', 'New York': 'NY', 'District of Columbia': 'DC',
    'Nevada': 'NV', 'Maine': 'ME'}

stateCodeList = [
    'MS', 'IA', 'OK', 'WY',
    'MN', 'IL', 'AR', 'NM',
    'IN', 'MD', 'LA', 'TX',
    'AZ', 'WI', 'MI', 'KS',
    'UT', 'VA', 'OR', 'CT',
    'MT', 'CA', 'MA', 'WV',
    'DE', 'NH', 'VT', 'GA',
    'ND', 'HI', 'PA', 'FL',
    'AK', 'KY', 'TN', 'SC',
    'NE', 'MO', 'OH', 'AL',
    'RI', 'SD', 'CO', 'ID',
    'NJ', 'WA', 'NC', 'NY',
    'NV', 'ME'
]

sa = open(data_path + 'state-address.json')
state_addresses = json.load(sa)

def gen_last_name(ucase=2, lcase=2,
                  compound_name=False, use_census_distribution=True):
    gen_name = {}

    ln = None
    cn = None
    if use_census_distribution:
        while ln is None:
            x = random.randrange(0, 905540)
            for (k, v) in last_names:
                if x > k:
                    ln = v
    else:
        while ln is None:
            x = random.randrange(len(last_names))
            ln = last_names[x][1]

    if compound_name == True:
        xc = random.randrange(0, 905540)
        # print(x)
        while cn is None:
            # try:
            #     cn = last_names[x]
            # except:
            #     x = x + 1
            #     pass
            if xc in last_names:
                cn = last_names[x]
            else:
                xc += 1
        ln = ln + '-' + cn
    u = random.randrange(0, 100)
    if u < ucase:
        gen_name['last_name'] = ln
        gen_name['case'] = 'u'
    elif u > 100 - lcase:
        gen_name['last_name'] = ln.swapcase()
        gen_name['case'] = 'l'
    else:
        gen_name['last_name'] =  ln.title()
        gen_name['case'] = 'p'
    gen_name['seed'] = x
    x = None
    return gen_name

def gen_first_name(ucase=2, lcase=2, gender=False):
    gen_name = {}
    ln = None
    x = random.randrange(0, 90040)
    while ln is None:
        try:
            if gender == 'f':
                ln = female_names[x]
            else:
                ln = male_names[x]
        except:
            x = x + 1
            pass
    u = random.randrange(0, 100)
    if u < ucase:
       gen_name['given_name'] = ln
       gen_name['case'] = 'u'
    elif u > 100 - lcase:
        gen_name['given_name'] = ln.swapcase()
        gen_name['case'] = 'l'
    else:
        gen_name['given_name'] = ln.title()
        gen_name['case'] = 'p'

    return gen_name

def gen_full_name(case=None, gender=None, gender_bias=GENDER_BIAS,
                  given_names=1, randomize_name_count = True):
    name = {}
    gns = []
    maiden_name = False
    compound_name = False
    if not gender:
        g = random.randrange(0, 99)
        if g <= gender_bias:
            name['gender'] = 'f'
        else:
            name['gender'] = 'm'
    else:
        name['gender'] = gender
    cn = random.randrange(1, 100)
    if cn > 95:
        compound_name = True
    gn = gen_last_name(compound_name = compound_name)
    name['surname'] = gn['last_name']

    name['first_to_last'] = gn['last_name']
    name['last_then_first'] = gn['last_name'] + ','

    if randomize_name_count:
        gnc = random.randrange(1, 100)
        if gender == 'm':
            if gnc < 50:
                given_names = 1
            elif gnc >= 50 and gnc <= 99:
                given_names = 2
            elif gnc == 100:
                given_names = 3
        else:
            if gnc < 70:
                given_names = 1
            elif gnc >= 70 and gnc <= 90:
                given_names = 2
                maiden_name = True
            elif gnc > 90 and gnc < 100:
                given_names = 2
            elif gnc == 100:
                given_names = 3
    names_list = "" # used to store the names.
    for x in range(given_names):
        if maiden_name and x > 0:
            #print 'Maiden'
            mn = gen_last_name(compound_name = False)
            nn =  {'given_name' : None, 'case' : None, 'gender' : None, 'ordinal' : 0  }
            nn['given_name'] = mn['last_name']
            nn['ordinal'] = x + 1
        else:
            nn = gen_first_name(gender=name['gender'])
            nn['ordinal'] = x + 1
        gns.append(nn)
        names_list = names_list + ' ' + nn['given_name']
    name['first_to_last'] = names_list + ' ' + name['surname']
    name['last_then_first'] = name['surname'] + ', ' + names_list.strip()
    name['given_names'] = gns
    gns = []
    return name

def gen_personal_email(first_name, last_name):
    domains = ['gmail.com', 'yahoo.com', 'hotmail.com',
               'icloud.com', 'aol.com', 'outlook.com']

    x = random.randrange(0, len(domains))

    f = random.randrange(0, 2)
    n = ''
    if f == 0:
        n = '{0}.{1}@'.format(first_name, last_name)
    elif f == 1:
        n = '{0}{1}@'.format(first_name[:1], last_name)
    else:
        n = '{0}{1}@'.format(first_name, last_name[:1])

    return n + domains[x]

def gen_address(state=None):
    # pkf = open(config.pickle_path + 'addresses.pkl', 'rb')
    # addresses = pickle.load(pkf)
    # pkf.close()
    # sublist = [address for address in addresses
    #         if address['state'] == state]
    # x = random.randrange(0, len(sublist))
    if state:
        if state in state_addresses:
            sc = len(state_addresses[state])
            x = random.randrange(0, sc - 1)
            # print(state_addresses[state])
            return state_addresses[state][x]
        else:
            raise ValueError('Unknown State Code')
    else:
        scl = random.randrange(0, len(stateCodeList))
        state = stateCodeList[scl]

        x = random.randrange(0, len(state_addresses[state]))
        return state_addresses[state][x]

    return address

def gen_employer(state):
    print(config.pickle_path)
    pkf = open(config.pickle_path +'employers.pkl', "rb")

    employers = pickle.load(pkf)
    pkf.close()

    sublist = [employer for employer in employers
            if employer['state'] == state]
    x = random.randrange(0, len(sublist))
    return sublist[x]

def gen_business_email(first, last, company):
    company = company.replace('&', ' ')
    company = company.replace('  ', ' ')
    y = company.split()
    if len(y) > 3:
        company = y[0] + y[1] #+ y[2]
    elif len(y) > 1:
        company = "".join([part for part in y if part != y[len(y) - 1]])
    else:
        company = y[0]
    company = company.replace(", Inc.", "").replace("Inc." , "").replace("Inc", "").replace('/', '')
    company = company.replace('|', '').replace(',', '').replace(';', '').replace("'", "").strip()
    if len(company) > 30:
        company = company[:20]
    email = gen_personal_email(first, last)
    email = email.split('@')[0] + '@' + company + '.com'
    email = email.replace('..', '.')
    return email

def gen_favorites(gender="f"):
    cuisine = ''
    dessert = ''
    music = ''
    snack = ''
    hobby = ''
    religion = ''
    drink = ''

    cuisines = ['American', 'Italian', 'Mexican', '???', 'French', 'Japanese',
                'Chinese', 'italian', 'Tex/Mex', 'Seafood', 'anything',
                'mexican', '', '', 'none', '' ]
    desserts = ['Apple Pie', 'Ice Cream', '', '', '', '', 'Cheesecake',
                'pumpkin pie', 'Yogurt', 'Fruit', 'Chocolate Cake']
    music_genres = ['Classical', 'jazz', 'Jazz', 'country', 'Country/Western',
                    'punk', 'punk-rock', 'disco', '', '', '', 'Pop', 'rock',
                    'Rock and Roll', '', 'Hip-Hop']
    religions = ['', '', '', '', '', '', 'Baptist', 'Catholic', 'Methodist',
                 'Jewish', 'atheist', 'None', 'Baptist', 'Lutheran', 'Unitarian',
                 'Presbyterian', 'catholic', 'Buddhist' ]
    snacks = ['peanuts', 'twinkies', 'Gummy Bears', "M&M's", 'pretzels', '',
              '', '', '', 'Chocolate Chip Cookies', 'Oreo''s', 'Swedish Fish',
              'Goldfish', 'Chocolate', 'Chocolate', 'chocolate', 'Candy',
              'Snickers', 'Apples', 'Bananas', 'banana', 'chips', 'almonds' ]
    drinks = ['Coffee', 'Coffee', 'coffee', 'tea', 'Tea', 'Red Bull', 'Monster',
              '', '', '', 'Coke', 'Coke Zero', 'Pepsi', 'Diet Coke', 'Coke',
              'Pepsi', 'Water', 'water', 'Orange Juice', 'Espresso', 'Ice Tea',
              'Lemonade']
    cuisine = cuisines[random.randrange(0, len(cuisines))]
    dessert = desserts[random.randrange(0, len(desserts))]
    music = music_genres[random.randrange(0, len(music_genres))]
    religion = religions[random.randrange(0, len(religions))]
    snack = snacks[random.randrange(0, len(snacks))]
    drink = drinks[random.randrange(0, len(drinks))]

    favorites = {'cuisine' : cuisine, 'dessert' : dessert,
                 'music' : music, 'snack' : snack, 'hobby' : hobby,
                 'religion' : religion, 'drink' : drink }

    return favorites

def gen_dates(birth_year=None):
    birthdate = None
    wedding = ''

    if birth_year:
        byear = random.randrange(birth_year - 5, birth_year + 5)
    else:
        byear = random.randrange(1944, 1992)
    birthdate = datetime.date(byear, random.randrange(1,12), random.randrange(1,28))

    wyear = random.randrange(byear + 18, byear + 35)

    if wyear > 2012: wyear  = 2012

    wedding = datetime.date(wyear, random.randrange(1,12), random.randrange(1, 28))

    results = {'birth' : birthdate, 'wedding' : wedding }

    return results

def gen_financials():
    net_worth = random.randrange(5, 33) * 10000
    liquid_assets = net_worth / random.randrange(1, 10)
    annual_income = random.randrange(7, 42) * 5000

    financials = {'net_worth' : net_worth, 'liquid_assets' : liquid_assets, 'annual_income' : annual_income }

    return financials

def gen_school(state):
    colleges = pickle.load(open(config.pickle_path + 'colleges.pkl', 'rb'))
    state_colleges = colleges[state]
    if state_colleges:
        return state_colleges[random.randrange(0, len(state_colleges))]

def gen_phone_number():
    area_code = random.randrange(100, 799)
    phone1 = random.randrange(100, 999)
    phone2 = random.randrange(1000, 9999)
    return str(area_code) + str(phone1) + str(phone2)

def gen_SSN():
    area_code = random.randrange(100, 799)
    phone1 = random.randrange(10, 99)
    phone2 = random.randrange(1000, 9999)
    return str(area_code) + "-" + str(phone1) + "-" + str(phone2)

def gen_bank_account():
    num_len = random.randrange(7, 12)
    x = 1
    for i in range(num_len):
        x = x * 10
    x = x - 1
    account_number = random.randrange(1, x)
    first_letter_seed = 22  #the percentage of account numbers with 1-2 initial letters.
    f = random.randrange(0, 99)
    if f <= first_letter_seed:
        account_number = 'AB' + str(account_number)
    return str(account_number)

def gen_drivers_license():
    num_len = random.randrange(7, 12)
    x = 1
    for i in range(num_len):
        x = x * 10
    x = x - 1
    account_number = random.randrange(1, x)
    first_letter_seed = 22  #the percentage of account numbers with 1-2 initial letters.
    f = random.randrange(0, 99)
    if f <= first_letter_seed:
        account_number = random.choice(string.ascii_letters).upper() + str(account_number)
    if f < (first_letter_seed / 2):
        account_number = random.choice(string.ascii_letters).upper() + str(account_number)
    return str(account_number)

def _luhn_total(num):
    digits = [int(x) for x in str(num)]
    total = sum(digits[-1::-2])
    for digit in digits[-2::-2]:
        digit *= 2
        if digit > 9: digit -= 9
        total += digit
    return total

def luhn_check(num):
    """Returns true if the number has a valid luhn checksum"""
    return _luhn_total(num) % 10 == 0

def luhn_gen(num):
    """Returns the check digit for the given number"""
    return (_luhn_total(num + "0") * 9) % 10

def _cc_range(ranges):
    """Generates a flat list of numbers from a list of individual numbers
        and (start, stop) pairs."""
    o_range = []
    for r in ranges:
        try:
            start, end = r
            o_range.extend(list(range(start, end+1)))
        except TypeError:
            o_range.append(r)
    return o_range

def cc(name, BINs, lengths, checksum=True):
    BINs = _cc_range(BINs)
    lengths = _cc_range(lengths)
    def _cc_gen():
        card_len = random.choice(lengths)
        BIN = str(random.choice(BINs))
        account_num_len = card_len - len(BIN)
        if checksum:
            account_num_len -= 1
        account_num = random.randrange(0, (10 ** (account_num_len + 1)) - 1)
        card_num = BIN + str(account_num)
        if checksum:
            card_num += str(luhn_gen(card_num))
        return card_num
    _cc_gen.__name__ = name
    return _cc_gen

# All information from: https://en.wikipedia.org/wiki/Payment_card_number
CC_TYPES_INACTIVE = [
    cc("Bankcard", [5610, (560221, 560225)], [16]),
    cc("Diners Club enRoute", [2014, 2149], [15], checksum=False),
    cc("Laser", [6304, 6706, 6771, 6709], [(16, 19)]),
    cc("MasterCard", [(2221, 2720)], [16]),
    cc("Solo", [6334, 6767], [16, 18, 19]),
    cc("Switch", [4903, 4905, 4911, 4936, 564182, 633110, 6333, 6759], [16, 18, 19])
]

CC_TYPES_ACTIVE = [
    cc("American Express", [34, 37], [16]),
    cc("China UnionPay", [62], [(16, 19)]),
    cc("Diners Club Carte Blanche", [(300, 305)], [14]),
    cc("Diners Club International",
            [(300, 305), 309, 36, 38, 39], [14]),
    cc("Discover Card",
            [6011, (622126, 622925), (644, 649), 65], [16, 19]),
    cc("InterPayment", [636], [(16, 19)]),
    cc("InstaPayment", [(637,639)], [16]),
    cc("JCB", [(3528, 3589)], [16]),
    cc("Maestro", [50, (56, 69)], [(12, 19)]),
    cc("Dankort", [5019], [16]),
    cc("Dankort + Visa", [4, 4175, 4571], [16]),
    cc("NSPK MIR", [(2200, 2204)], [16]),
    cc("MasterCard", [(51, 55)], [16]),
    cc("Visa", [4], [13, 16, 19]),
    cc("UATP", [1], [15]),
    cc("Verve", [(506099, 506198), (650002, 650027)], [16, 19]),
    cc("CARDGUARD EAD BG ILS", [5392], [16])
]

# There could be many better sampling methods, for example sampling according
# to popular distribution of cards. I imagine there are many more visas than
# "InterPayment" cards, at least in the U.S.
# Even sampling according to the possible number of unique cards would probably
# be better.
def gen_credit_card_number():
    return random.choice(CC_TYPES_ACTIVE)()

if __name__ == '__main__':
    # print('10 random names')
    # for i in range(10):
    #     print(gen_full_name())
    # print('10 names matching distribution')
    for i in range(10):
        print(gen_bank_account())


'''
Powerbot demo signals script
(c) 2018 Inercomp GmbH
'''

import os, logging, sys, json, random
from datetime import datetime,timedelta
from powerbot import Configuration, ApiClient
from powerbot.api import SignalsApi

## Setting up logging and defining global variables
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger()

#Demo script accepts api_key as commandline parameter or as an environmental variable
api_key = str(sys.argv[1]) if len(sys.argv) >= 2 else os.getenv('POWERBOT_PLAYGROUND_API_KEY')
url = str(sys.argv[2]) if len(sys.argv) >= 3 else os.getenv('POWERBOT_PLAYGROUND_URL')

if not url or len(url.strip()) == 0:
    url = "https://playground.powerbot-trading.com/api/v0"

if not api_key or len(api_key.strip()) == 0:
    raise ValueError("Please provide your api_key on the command line or as the 'POWERBOT_PLAYGROUND_API_KEY' environmen variable")

# Retrieving handles to the powerbot api
client = ApiClient()
client.set_default_header("api_key",api_key)

signals = SignalsApi(client)

hour = 0
now = datetime.utcnow()

LOGGER.info("Starting signals against {} with api_key {}*****".format(url,api_key[:5]))

while hour <= 24:
    hour+=1

    # Submit a demo ETRMSystem signal with a random imbalance between -10 and +10 MW
    delivery_start = now.replace(minute=0, second=0, microsecond=0) + timedelta(hours=hour)
    varImbalance = round(random.uniform(-10,10), 2)
    signals.update_signal(
        id=delivery_start.strftime("etrm_signal_%Y-%m-%dT%H"),
		source = "ETRMSystem",
		delivery_start = delivery_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
		delivery_end = (delivery_start + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ"),
        value = {
            "imbalance": varImbalance
		}
    )

    # Submit a demo OptSystem signal with a random marginal_price between 40 and 50
    varMarginalPrice = round(random.uniform(40,50), 2)
    signals.update_signal(
        id=delivery_start.strftime("opt_signal_%Y-%m-%dT%H"),
        source = "OptSystem",
        delivery_start = delivery_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
        delivery_end = (delivery_start + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ"),
        value = {
            "marginal_price": varMarginalPrice
        }
    )

	# Mirror the signals to show in GUI chart by sending them as source "display"
    signals.update_signal(
        id=delivery_start.strftime("display_signal_%Y-%m-%dT%H"),
        source = "display",
        delivery_start = delivery_start.strftime("%Y-%m-%dT%H:%M:%SZ"),
        delivery_end = (delivery_start + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ"),
        value = {
            "marginal_price": varMarginalPrice,
            "imbalance": varImbalance
        }
    )

LOGGER.info("Signals completed.")


from django.shortcuts import render, redirect
from .forms import AddReview
from .models import Review
from django.utils import timezone

# Create your views here.
def ReviewCreate(request):
    form = AddReview()
    if request.method == "POST":
        form = AddReview(request.POST)
        if form.is_valid():
            review = form.save(commit=False)
            review.created_date = timezone.now()
            if 'attachment' in request.FILES:
                review.attachment = request.FILES['attachment']
            review.save()
            return redirect('feedback/ty.html')
        else:
            form = AddReview()
    return render(request, 'feedback/add_review.html', {'form': form, })


import numpy as np
import phase as phase

class BasisGenerator():

    def __init__(self, phaseGenerator, numBasis = 10):

        self.numBasis = numBasis
        self.phaseGenerator = phaseGenerator

    def basis(self):

        return None

    def basisMultiDoF(self, time, numDoF):
        #print('time in=', time)
        basisSingleDoF = self.basis(time)  # get a sampled basis function for a single DoF, is the one in NormalizedRBFBasisGenerator(BasisGenerator) class, is 1 dim if time is 1dim and basissingle is nb of basis/dof x nb of samples (time samples0)
        #print('basisSingleDoF_dim=', basisSingleDoF.shape) # 1x5
        basisMultiDoF = np.zeros((basisSingleDoF.shape[0] * numDoF, basisSingleDoF.shape[1] * numDoF)) # basisSingleDoF.shape[0] gives number of basis fns for 1 dof, and shape[1] gives nb of basis/time samples ? basisMultiDoF is a block diag matrix, to sum up it's 3x15

        for i in range(numDoF): # 0, 1, 2
            rowIndices = slice(i * basisSingleDoF.shape[0], (i + 1) * basisSingleDoF.shape[0]) # indices from start to stop
            columnIndices = slice(i * basisSingleDoF.shape[1], (i + 1) * basisSingleDoF.shape[1])

            basisMultiDoF[rowIndices, columnIndices] = basisSingleDoF
        return basisMultiDoF

class DMPBasisGenerator(BasisGenerator):

    def __init__(self, phaseGenerator, numBasis = 10, duration = 1, basisBandWidthFactor = 3):
        BasisGenerator.__init__(self, phaseGenerator, numBasis)

        self.basisBandWidthFactor = basisBandWidthFactor

        timePoints = np.linspace(0, duration, self.numBasis)
        self.centers = self.phaseGenerator.phase(timePoints)

        tmpBandWidth = np.hstack((self.centers[1:]-self.centers[0:-1], self.centers[-1] - self.centers[- 2]))

        # The Centers should not overlap too much (makes w almost random due to aliasing effect).Empirically chosen
        self.bandWidth = self.basisBandWidthFactor / (tmpBandWidth ** 2)

    def basis(self, time):

        phase = self.phaseGenerator.phase(time)

        diffSqr = np.array([((x - self.centers) ** 2) * self.bandWidth for x in phase])
        basis = np.exp(- diffSqr /  2)

        sumB = np.sum(basis, axis=1)
        basis = [column * phase / sumB for column in basis.transpose()]
        return np.array(basis).transpose()

class NormalizedRBFBasisGenerator(BasisGenerator):

    def __init__(self, phaseGenerator, numBasis = 10, duration = 1, basisBandWidthFactor = 3, numBasisOutside = 0):
        BasisGenerator.__init__(self, phaseGenerator, numBasis)

        self.basisBandWidthFactor = basisBandWidthFactor
        self.numBasisOutside = numBasisOutside
        basisDist = duration / (self.numBasis - 2 * self.numBasisOutside - 1)  # dist between 2 consecutive basis in temporal space, numBasis provided is total including the nb outside the duration (1s) range and is the number on a one side of that duration (right or left)

        timePoints = np.linspace(-self.numBasisOutside * basisDist, duration + self.numBasisOutside * basisDist, self.numBasis)  # time for the center points of the basis to generate
        self.centers = self.phaseGenerator.phase(timePoints)

        tmpBandWidth = np.hstack((self.centers[1:]-self.centers[0:-1], self.centers[-1] - self.centers[- 2]))  # ???

        # The Centers should not overlap too much (makes w almost random due to aliasing effect).Empirically chosen
        self.bandWidth = self.basisBandWidthFactor / (tmpBandWidth ** 2) # bandwidth is the spatial scale of the Basis functions

    def basis(self, time):

        if isinstance(time, (float, int)):
            time = np.array([time])

        phase = self.phaseGenerator.phase(time)  # z(t)

        diffSqr = np.array([((x - self.centers) ** 2) * self.bandWidth for x in phase])
        basis = np.exp(- diffSqr / 2) # gaussian basis functions , h = given here as 1 / basiswidth

        sumB = np.sum(basis, axis=1) # axis = 1  --- > column
        basis = [column / sumB for column in basis.transpose()]
        return np.array(basis).transpose()  # each basis generate dat this level is along column, each basis is sampled along 1 column

class NormalizedRhythmicBasisGenerator(BasisGenerator):

    def __init__(self, phaseGenerator,  numBasis = 10, duration = 1, basisBandWidthFactor = 3):

        BasisGenerator.__init__(self, phaseGenerator, numBasis)

        self.numBandWidthFactor = basisBandWidthFactor
        self.centers = np.linspace(0, 1, self.numBasis)

        tmpBandWidth = np.hstack((self.centers[1:]-self.centers[0:-1], self.centers[-1] - self.centers[- 2]))

        # The Centers should not overlap too much (makes w almost random due to aliasing effect).Empirically chosen
        self.bandWidth = self.basisBandWidthFactor / (tmpBandWidth ** 2)

    def basis(self):

        phase = self.getInputTensorIndex(0)

        diff = np.arraay([np.cos((phase - self.centers) * self.bandWidth * 2 * np.pi)])
        basis = np.exp(diff)

        sumB = np.sum(basis, axis=1)
        basis = [column / sumB for column in basis.transpose()]
        return np.array(basis).transpose()

if __name__ == "__main__":
    import matplotlib.pyplot as plt
    import numpy as np

    phaseGenerator = phase.LinearPhaseGenerator()
    basisGenerator = NormalizedRBFBasisGenerator(phaseGenerator, numBasis = 10, duration = 1, basisBandWidthFactor = 3, numBasisOutside = 1)

    time = np.linspace(0, 1, 100)
    basis = basisGenerator.basis(time)
    basisMultiDoF = basisGenerator.basisMultiDoF(time, 3)

    plt.figure()
    plt.plot(time, basis)

    plt.show()

    print('done')


import numpy as np
import scipy
import math
import matplotlib.pyplot as plt
from scipy import special
from scipy.fftpack import fft, ifft

#   http://www.satbroadcasts.com/DVB-S_Bitrate_and_Bandwidth_Calculator.html

symrate = 27500.0
symdur = 1/symrate
poi_per_sym = 100
k = 10.0/symdur

#zero_zero = np.sinc(k*(n-symdur*0.125)) #time domain
#zero_one = np.sinc(k*(n-symdur*0.375))
#one_zero = np.sinc(k*(n-symdur*0.625))
#one_one = np.sinc(k*(n-symdur*0.875))

h=[]

f = np.linspace(-1e4,1e4,int(poi_per_sym))
Ts = 1e-2 # what values should be taken  ?
fn = 1/(2*Ts)
alpha = 0.35

def func(x,alpha):
	if abs(x) < fn*(1-alpha) :
		return 1
	elif abs(x) > fn*(1+alpha) :
		return 0
	else:
		return np.sqrt(0.5+0.5*(math.sin((math.pi/(2*fn))*((fn-abs(x))/alpha))))

def genh(alpha):
	for i in range(0,int(poi_per_sym)):
		h.append(func(f[i],alpha))

genh(alpha)

noof_symbols = 30

n = np.linspace(0,noof_symbols*symdur,noof_symbols*poi_per_sym)

xbinary = np.random.randint(2, size = 2*noof_symbols)
xtime = np.zeros(noof_symbols*poi_per_sym)

for i in range(0,noof_symbols):
	if xbinary[2*i] == 0 and xbinary[2*i + 1] == 0 :
		xtime = xtime + np.sinc(k*(n-(i+1)*symdur*0.125))
	elif xbinary[2*i] == 0 and xbinary[2*i + 1] == 1 :
		xtime = xtime + np.sinc(k*(n-(i+1)*symdur*0.375))
	elif xbinary[2*i] == 1 and xbinary[2*i + 1] == 0 :
		xtime = xtime + np.sinc(k*(n-(i+1)*symdur*0.625))
	elif xbinary[2*i] == 1 and xbinary[2*i + 1] == 1 :
		xtime = xtime + np.sinc(k*(n-(i+1)*symdur*0.875))

xtime_with_noise = xtime + np.random.normal(0,1,len(xtime))
xfft = fft(xtime_with_noise)
yfft = h*xfft
ytime = ifft(yfft)

# how to get xhattime from yfft ?

# Here part

plt.plot(n,xtime,label='xtime')
plt.plot(n,abs(fft(xtime)),label='fft')
plt.legend(loc='best')

plt.grid()
plt.show()


from __future__ import division

import os
import time
import numpy as np
import tensorflow as tf

from ops import batch_norm, lrelu, conv2d, dilated_conv2d, conv2d_transpose, linear
from utils import dataset_files, get_image

D_ITERATIONS = 4

class WGANTensor(object):
    def __init__(self,
                 sess,
                 learning_rate=0.0001,
                 beta1=0.0,
                 beta2=0.5,
                 batch_size=32,
                 image_size=64,
                 lam=100,
                 checkpoint_dir=None):
        self.sess = sess
        self.batch_size = batch_size
        self.image_size = image_size
        self.image_shape = [image_size, image_size, 3]
        self.lam = lam
        self.checkpoint_dir = checkpoint_dir
        self.global_step = tf.Variable(0, trainable=False)
        self.d_bns = [batch_norm(name='d_bn{}'.format(i)) for i in range(10)]
        self.g_bns = [batch_norm(name='g_bn{}'.format(i, )) for i in range(15)]

        self.images = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='images')
        self.masks = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='masks')
        self.masked_images = tf.multiply(self.images, self.masks)
        self.G = self.generator(self.masked_images)
        self.G_final = self.masked_images + tf.multiply(self.G, (1 - self.masks))

        self.d_loss_real = tf.reduce_mean(self.discriminator(self.images))
        self.d_loss_fake = tf.reduce_mean(self.discriminator(self.G, reuse=True))
        self.d_loss = self.d_loss_fake - self.d_loss_real
        self.g_loss_mse = tf.reduce_mean(tf.contrib.layers.flatten(tf.square(self.G - self.images)))
        self.g_loss = self.lam * self.g_loss_mse - self.d_loss_fake

        self.epsilon = tf.random_uniform([], 0.0, 1.0)
        self.G_epsilon = self.epsilon * self.images + (1 - self.epsilon) * self.G
        self.D_epsilon = self.discriminator(self.G_epsilon, reuse=True)
        self.gradients = tf.gradients(self.D_epsilon, self.G_epsilon)[0]
        self.slopes = tf.sqrt(tf.reduce_sum(tf.square(self.gradients), reduction_indices=[1]))
        self.gradient_penalty = 100.0 * tf.reduce_mean((self.slopes - 1.0) ** 2)
        self.d_loss += self.gradient_penalty

        self.d_loss_real_summary = tf.summary.scalar("d_loss_real", self.d_loss_real)
        self.d_loss_fake_summary = tf.summary.scalar("d_loss_fake", self.d_loss_fake)
        self.gradient_penalty_summary = tf.summary.scalar("d_loss_gp", self.gradient_penalty)
        self.d_loss_summary = tf.summary.scalar("d_loss", self.d_loss)
        self.d_summary = tf.summary.merge([
            self.d_loss_real_summary, self.d_loss_fake_summary, self.gradient_penalty_summary, self.d_loss_summary])

        self.masked_images_summary = tf.summary.image("masked_image", self.masked_images)
        self.G_summary = tf.summary.image("g", self.G)
        self.G_final_summary = tf.summary.image("g_final", self.G_final)
        self.g_loss_mse_summary = tf.summary.scalar("g_mse", self.g_loss_mse)
        self.g_loss_summary = tf.summary.scalar("g", self.g_loss)
        self.g_summary = tf.summary.merge([
            self.G_summary, self.masked_images_summary, self.G_final_summary,
            self.g_loss_mse_summary, self.g_loss_summary])

        t_vars = tf.trainable_variables()
        self.d_vars = [var for var in t_vars if 'd_' in var.name]
        self.g_vars = [var for var in t_vars if 'g_' in var.name]
        self.d_optim = tf.train.AdamOptimizer(learning_rate, beta1=beta1, beta2=beta2).minimize(
            self.d_loss, var_list=self.d_vars, global_step=self.global_step)
        self.g_optim = tf.train.AdamOptimizer(learning_rate, beta1=beta1, beta2=beta2).minimize(
            self.g_loss, var_list=self.g_vars)

        self.writer = tf.summary.FileWriter(os.path.join(self.checkpoint_dir, "logs"), self.sess.graph)
        self.saver = tf.train.Saver(max_to_keep=10)

    def train(self, config):
        tf.global_variables_initializer().run()
        start_time = time.time()
        if self.load(self.checkpoint_dir):
            print("""Loading existing model""")
        else:
            print("""Initalizing new model""")

        for epoch in range(config.epoch):
            data = dataset_files(config.dataset)
            np.random.shuffle(data)
            batch_idx = min(len(data), config.train_size) // self.batch_size
            for idx in range(batch_idx):
                batch_files = data[idx * config.batch_size: (idx + 1) * config.batch_size]
                batch = [get_image(batch_file, self.image_size) for batch_file in batch_files]
                batch_images = np.array(batch).astype(np.float32)
                masks = np.ones([config.batch_size, config.image_size, config.image_size, 3])
                masks[:, 16:48, 16:48, :] = 0.0

                _, d_loss, summary_str = self.sess.run(
                    [self.d_optim, self.d_loss, self.d_summary],
                    feed_dict={self.images: batch_images, self.masks: masks})
                self.writer.add_summary(summary_str, self.global_step.eval())

                if idx > 0 and idx % D_ITERATIONS == D_ITERATIONS - 1:
                    _, g_loss, summary_str = self.sess.run(
                        [self.g_optim, self.g_loss, self.g_summary],
                        feed_dict={self.images: batch_images, self.masks: masks})
                    self.writer.add_summary(summary_str, self.global_step.eval())
                    print("Epoch: [{:2d}] [{:4d}/{:4d}] time: {:4.4f}, d_loss: {:.8f}, g_loss: {:.8f}".format(
                        epoch, idx, batch_idx, time.time() - start_time, d_loss, g_loss))

                if idx % 100 == 0:
                    self.save(config.checkpoint_dir, self.global_step.eval())

    def generator(self, images):
        with tf.variable_scope("generator"):
            g_h0 = tf.nn.relu(conv2d(images, 64, 5, 5, 1, 1, name='g_h0'))
            g_h1 = tf.nn.relu(self.g_bns[0](conv2d(g_h0, 128, 3, 3, 2, 2, name='g_h1')))
            g_h2 = tf.nn.relu(self.g_bns[1](conv2d(g_h1, 128, 3, 3, 1, 1, name='g_h2')))
            g_h3 = tf.nn.relu(self.g_bns[2](conv2d(g_h2, 256, 3, 3, 2, 2, name='g_h3')))
            g_h4 = tf.nn.relu(self.g_bns[3](conv2d(g_h3, 256, 3, 3, 1, 1, name='g_h4')))
            g_h5 = tf.nn.relu(self.g_bns[4](conv2d(g_h4, 256, 3, 3, 1, 1, name='g_h5')))
            g_h6 = tf.nn.relu(self.g_bns[5](dilated_conv2d(g_h5, 256, 3, 3, 2, name='g_h6')))
            g_h7 = tf.nn.relu(self.g_bns[6](dilated_conv2d(g_h6, 256, 3, 3, 4, name='g_h7')))
            g_h8 = tf.nn.relu(self.g_bns[7](dilated_conv2d(g_h7, 256, 3, 3, 8, name='g_h8')))
            g_h9 = tf.nn.relu(self.g_bns[8](dilated_conv2d(g_h8, 256, 3, 3, 16, name='g_h9')))
            g_h10 = tf.nn.relu(self.g_bns[9](conv2d(g_h9, 256, 3, 3, 1, 1, name='g_h10')))
            g_h11 = tf.nn.relu(self.g_bns[10](conv2d(g_h10, 256, 3, 3, 1, 1, name='g_h11')))
            g_h12 = tf.nn.relu(self.g_bns[11](conv2d_transpose(
                g_h11, [self.batch_size, int(self.image_size/2), int(self.image_size/2), 128], 4, 4, 2, 2, name='g_h12')))
            g_h13 = tf.nn.relu(self.g_bns[12](conv2d(g_h12, 128, 3, 3, 1, 1, name='g_h13')))
            g_h14 = tf.nn.relu(self.g_bns[13](conv2d_transpose(
                g_h13, [self.batch_size, self.image_size, self.image_size, 64], 4, 4, 2, 2, name='g_h14')))
            g_h15 = tf.nn.relu(self.g_bns[14](conv2d(g_h14, 32, 3, 3, 1, 1, name='g_h15')))
            g_h16 = tf.nn.tanh(conv2d(g_h15, 3, 3, 3, 1, 1, name='g_h16'))
            return g_h16

    def discriminator(self, images, reuse=False):
        with tf.variable_scope('discriminator', reuse=reuse):
            gd_h0 = lrelu(conv2d(images, 64, name="d_gd_h0_conv")) # 32*32*64
            gd_h1 = lrelu(self.d_bns[0](conv2d(gd_h0, 128, name='d_gd_h1_conv'))) # 16*16*128
            gd_h2 = lrelu(self.d_bns[1](conv2d(gd_h1, 256, name='d_gd_h2_conv'))) # 8*8*256
            gd_h3 = lrelu(self.d_bns[2](conv2d(gd_h2, 512, name='d_gd_h3_conv'))) # 4*4*512
            gd_fc0 = linear(tf.reshape(gd_h3, [self.batch_size, int(4 * 4 * 512)]), 1024, 'd_gd_fc0')
            gd_fc1 = linear(gd_fc0, 1, 'd_gd_fc1')
            return gd_fc1

    def save(self, checkpoint_dir, step):
        if not os.path.exists(checkpoint_dir):
            os.makedirs(checkpoint_dir)
        self.saver.save(self.sess,
                        os.path.join(checkpoint_dir, "wgan_tensor"),
                        global_step=step)

    def load(self, checkpoint_dir):
        print(" [*] Reading checkpoints...")
        ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
        if ckpt and ckpt.model_checkpoint_path:
            self.saver.restore(self.sess, ckpt.model_checkpoint_path)
            return True
        else:
            return False

flags = tf.app.flags
flags.DEFINE_integer("epoch", 25, "Epoch to train [25]")
flags.DEFINE_float("learning_rate", 0.0001, "Learning rate of for adam [0.0001]")
flags.DEFINE_float("beta1", 0.0, "Momentum term of adam [0.0]")
flags.DEFINE_float("beta2", 0.9, "Beta 2 of adam [0.9]")
flags.DEFINE_integer("train_size", np.inf, "The size of train images [np.inf]")
flags.DEFINE_integer("batch_size", 64, "The size of batch images [32]")
flags.DEFINE_integer("image_size", 64, "The size of image to use")
flags.DEFINE_string("dataset", "lfw-aligned-64", "Dataset directory.")
flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoint]")
flags.DEFINE_string("sample_dir", "samples", "Directory name to save the image samples [samples]")
FLAGS = flags.FLAGS

if not os.path.exists(FLAGS.checkpoint_dir):
    os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
    os.makedirs(FLAGS.sample_dir)

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
    model = WGANTensor(
        sess,
        learning_rate=FLAGS.learning_rate,
        beta1=FLAGS.beta1,
        beta2=FLAGS.beta2,
        image_size=FLAGS.image_size,
        batch_size=FLAGS.batch_size,
        checkpoint_dir=FLAGS.checkpoint_dir)
    model.train(FLAGS)


import os
import math
import requests
from pureapi import response

pure_api_url = os.environ.get('PURE_API_URL')
pure_api_key = os.environ.get('PURE_API_KEY')

headers = {
  'Accept': 'application/json',
  'api-key': pure_api_key,
}

def get(endpoint, params={}, headers=headers):
  return requests.get(
    pure_api_url + endpoint,
    params=params,
    headers=headers,
  )

def get_all(endpoint, params={}, headers=headers):
  r = get(endpoint, {'size': 0, 'offset': 0}, headers)
  json = r.json()
  record_count = int(json['count'])
  window_size = int(params['size']) if 'size' in params else 100
  window_count = int(math.ceil(float(record_count) / window_size))

  for window in range(0, window_count):
     window_params = {'offset': window * window_size, 'size': window_size}
     yield get(endpoint, {**params, **window_params}, headers)

def get_all_transformed(endpoint, params={}, headers=headers):
  for r in get_all(endpoint, params, headers):
    for item in r.json()['items']:
      yield response.transform(endpoint, item)

def filter(endpoint, payload={}, headers=headers):
  return requests.post(
    pure_api_url + endpoint,
    json=payload,
    headers=headers,
  )

def filter_all(endpoint, payload={}, headers=headers):
  count_payload = payload
  count_payload['size'] = 0
  count_payload['offset'] = 0
  r = filter(endpoint, count_payload, headers)
  json = r.json()
  record_count = int(json['count'])
  window_size = int(payload.setdefault('size', 100))
  if window_size >= 0:
    window_size = 100
  payload['size'] = window_size
  window_count = int(math.ceil(float(record_count) / window_size))

  for window in range(0, window_count):
     payload['offset'] = window * window_size
     yield filter(endpoint, payload, headers)

def filter_all_transformed(endpoint, payload={}, headers=headers):
  for r in filter_all(endpoint, payload, headers):
    for item in r.json()['items']:
      yield response.transform(endpoint, item)


import boto3
import time
import datetime
#import picamera import PiCamera

from botocore.exceptions import NoCredentialsError

# reference your own access and secret keys
ACCESS_KEY = ""
SECRET_KEY = ""

def upload_to_aws(local_file, bucket, s3_file):
    s3 = boto3.client('s3', aws_access_key_id=ACCESS_KEY,
                      aws_secret_access_key=SECRET_KEY)

    try:
        s3.upload_file(local_file, bucket, s3_file)
        print("Upload Successful")
        return True
    except FileNotFoundError:
        print("The file was not found")
        return False
    except NoCredentialsError:
        print("Credentials not available")
        return False

def main():
    camera = PiCamera()
    camera.resolution = (1024, 768)
    camera.start_preview()
    time.sleep(2) # warmup time
    while(True):
        ts = time.time()
        st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
        camera.capture(str(st))
        time.sleep(2)
        upload_to_aws(st, 'iot-corbi', st)
        time.sleep(2)

if __name__ == '__main__':
    main()


import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
from scipy.signal import fftconvolve
import torch

def get_arg_support(dt, t_support, t0=0):
    arg_support0 = int((t_support[0] - t0) / dt)
    arg_supportf = int(np.ceil((t_support[1] - t0) / dt))
    return arg_support0, arg_supportf

def get_dt(t):
    arg_dt = 20 if len(t) >= 20 else len(t)
    dt = np.median(np.diff(t[:arg_dt]))
    return dt

def plot_spiketrain(t, mask_spikes, ax=None, **kwargs):

    color = kwargs.get('color', 'C0')
    marker = kwargs.get('marker', 'o')
    ms = kwargs.get('ms', mpl.rcParams['lines.markersize'])
    mew = kwargs.get('mew', 0)
    label = kwargs.get('label', None)

    no_ax = False
    if ax is None:
        figsize = kwargs.get('figsize', (6, 2))
        fig, ax = plt.subplots(figsize=figsize)
        ax.set_xlabel('time (ms)')
        no_ax = True

    arg_spikes = np.where(mask_spikes)
    ax.plot(t[arg_spikes[0]], arg_spikes[1], marker=marker, lw=0, color=color, ms=ms, mew=mew)

    extra_xspace = (t[-1] - t[0]) * .01
    ax.set_xlim(t[0] - extra_xspace, t[-1] + extra_xspace)
    ax.tick_params('y', labelleft=False, left=False)
    if label is not None:
        ax.set_ylabel(label)

    return ax

def raw_autocorrelation(mask_spikes, padding=None):
    padding = padding if padding is not None else mask_spikes.shape[0]
    x = mask_spikes.numpy()
    autocor = fftconvolve(x, x[::-1], mode='full', axes=0)[::-1] / x.shape[0]
    arg_lag0 = autocor.shape[0] // 2
    autocor = torch.from_numpy(autocor[arg_lag0:arg_lag0 + padding])
    return autocor

def searchsorted(t, s, side='left'):
    '''
    Uses np.searchsorted but handles numerical round error with care
    such that returned index satisfies
    t[i-1] < s <= t[i]
    np.searchsorted(side='right') doesn't properly handle the equality sign
    on the right side
    '''
    s = np.atleast_1d(s)
    arg = np.searchsorted(t, s, side=side)

    if len(t) > 1:
        dt = get_dt(t)
        s_ = (s - t[0]) / dt
        round_s = np.round(s_, 0)
        mask_round = np.isclose(s_, np.round(s_, 0)) & (round_s >= 0) & (round_s < len(t))
        if side == 'left':
            arg[mask_round] = np.array(round_s[mask_round], dtype=int)
        elif side == 'right':
            arg[mask_round] = np.array(round_s[mask_round], dtype=int) + 1
    else:
        s_ = s - t[0]
        mask = np.isclose(s - t[0], 0.)# & (round_s >= 0) & (round_s < len(t))
        arg[mask] = np.array(s_[mask], dtype=int)

    if len(arg) == 1:
        arg = arg[0]

    return arg

def shift_array(arr, shift, fill_value=False):
    """
    Shifts array on axis 0 filling the shifted values with fill_value
    Positive shift is to the right, negative to the left
    """

    result = np.empty_like(arr)
    if shift > 0:
        result[:shift, ...] = fill_value
        result[shift:, ...] = arr[:-shift, ...]
    elif shift < 0:
        result[shift:, ...] = fill_value
        result[:shift, ...] = arr[-shift:, ...]
    else:
        result = arr
    return result


import pymongo
import bcrypt
import datetime
from pymongo import MongoClient
import json

class LoginModel:

    def __init__(self):
        try:
            self.client = MongoClient()
            self.db     = self.client.social_net_db
            self.Users  = self.db.users
            print("Connected successfully!!!")
        except:
            print("Could not connect to MongoDB")

    def  login_user(self, data):

        user = self.Users.find_one({"username":data.username})

        if user!=None:
            if bcrypt.checkpw(data.password.encode(), user["password"].encode()):
                return user["username"]
            else:
                return False
        else:
            return False