# -*- coding: utf-8 -*-

# import win32com.client as win32
import urllib.request, json
import requests
from bs4 import BeautifulSoup
from emailtemplates import get_existing_templates

def getGovDetails(postcode):

    url_base = "http://api.postcodes.io/postcodes/"
    with urllib.request.urlopen(url_base + postcode) as url:
        data = json.loads(url.read().decode())
        if data["status"] == 200:
            topdata = data["result"]
        else:
            raise KeyError("No postcode found!")

    MPurl = (
        "http://lda.data.parliament.uk/commonsmembers.json?_view=members&_pageSize=2097&_page=0&constituency.label="
        + "%20".join(topdata["parliamentary_constituency"].split(" "))
    )

    with urllib.request.urlopen(MPurl) as url:
        MPdata = json.loads(url.read().decode())

        for possibleMP in MPdata["result"]["items"]:
            MPid = (possibleMP["_about"]).split("/")[-1]
            print("Checking MP: {}".format(possibleMP["fullName"]))

            try:
                MPurl = "https://members.parliament.uk/member/{}/contact".format(MPid)
                MPemails = emailExtractor(MPurl)  # MP email (in a list)
                assert len(MPemails) > 0
                # Quick hack to only return one email
                MPemail = MPemails[0]

                myward = topdata["admin_ward"]  # User's ward
                MPname = possibleMP["fullName"]["_value"]
                print("Found correct MP: {}. Email: {} ".format(MPname, MPemail))
                break
            except:
                pass

    return {"ward": myward, "MPemail": MPemail, "MPname": MPname}

def emailExtractor(urlString):
    emailList = []
    getH = requests.get(urlString)
    h = getH.content
    soup = BeautifulSoup(h, "html.parser")
    mailtos = soup.select("a[href^=mailto]")
    for i in mailtos:
        href = i["href"]
        try:
            str1, str2 = href.split(":")
        except ValueError:
            break

        emailList.append(str2)
    return emailList

def draftEmails(myname, postcode):
    ret = getGovDetails(postcode)
    ward = ret["ward"]
    MPname = ret["MPname"]
    MPemail = ret["MPemail"]

    print(
        "Details found. You live in {} ward and your MP is {}, with email: {}".format(
            ward, MPname, MPemail
        )
    )

    # Empty list of filled templates
    filled_email_templates = []
    # Get all the empty templates from templates.py
    empty_email_templates = get_existing_templates()
    # Set the user dictionary to include the name of the person sending
    user = {"name": myname}

    for e in empty_email_templates:  # For each empty template
        if e.target is None:
            # If no defined target, use MP info to fill target fields
            e.set_target(name=MPname, email=MPemail, ward=ward)

        # ToDo : Implement setting a cc

        # Pass the dictionary containing user information to the template filler
        success = e.fill(user)  # Returns true if successfully filled

        if success:
            # Append successful templates to the list we return
            filled_email_templates.append(e)
        else:
            print("Failed to fill template, subject: {}".format(e.subject))
            pass

    return filled_email_templates


### This program presents the pictorial depiction of the impact on the weights of the features based on the different values of
### regularilization parameters. In this case we used Logistic regression.
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import Machine_Learning.CommonFunctions.commonvariables as common
import sys

colors=common.colors
weights,params = [],[]
df_wine=common.df_wine
x_train_std=common.x_train_std
x_test_std=common.x_test_std
y_train=common.y_train

fig=plt.figure()
ax=plt.subplot(111)

## Execute the Logistic Regression  using L1 regularization (lasso) with different Regularization factor.
## In this C is the factor. Penalty =l1 means L1 regularization.
for c in np.arange(-4,6):
    lr=LogisticRegression(penalty='l1',C=10**c,random_state=0)
    lr.fit(x_train_std,y_train)
    weights.append(lr.coef_[1])
    params.append(10**c)

weights = np.array(weights)
for column,color in zip(range(weights.shape[1]),colors):
    #print ("{},{}".format(column,color))
    plt.plot(params,weights[:,column],label=df_wine.columns[column+1],color=color)
    #print ("{},{}".format(params,weights[:,column]))
    #plt.show()

plt.axhline(0,color='black',linestyle='--',linewidth=3)
#plt.xlim([10**(-5),10**(-6)])
plt.xlabel('C')
plt.ylabel('Weight Coeffients')
plt.xscale('log')
plt.legend(loc='upper left')
ax.legend(loc='upper center',bbox_to_anchor=(1.38,1.03),ncol=1,fancybox=True)

plt.show()

Machine_Learning/4 - Data Preprocessing/ReguarizationPath.py


'''
练习1：用for循环实现1~100之间的偶数求和
'''
sum = 0
for i in range(1, 100, 2):
    sum += i
print('1-100之间的偶数之和为：', sum)
# (1,100)只能取到1和99，不包含100；取偶数是从2开始，我们可以写成（2,101,2）

sum = 0
for i in range(2, 101):
    if i % 2 == 0:
        sum += i
print('1-100之间的偶数之和为：', sum)

'''
练习2:计算机出一个1到100之间的随机数，玩家输入自己猜的数字，计算机给出对应的提示信息（大一点、小一点或猜对了），如果玩家猜中了数字，计算机提示用户一共猜了多少次，游戏结束，否则游戏继续。
'''
import random

rand = random.randint(1, 100)
count = 0
while True:
    count += 1
    user = int(input('输入数字：'))
    if rand > user:
        print('大一点')
    elif rand < user:
        print('小一点')
    else:
        print('猜对了')
        break
print('一共猜了%d次' % count)

'''
练习3：输出乘法口诀表(九九表)
'''
for i in range(1, 10):
    for j in range(1, i + 1):
        print('%d*%d=%d' % (i, j, i * j), end='\t')
    print()

'''
练习4：输入一个正整数判断是不是素数。
提示：素数指的是只能被1和自身因子的整数。
'''
import math
number = int(input('输入一个正整数：'))
seq=int(math.sqrt(number))
is_prime=True
for i in range(2,seq+1):
    if number % i ==0:
        is_prime=False
        break
# 先判断这个数被它的平方根是否能够整除，如果不能，就是true
if  is_prime is True and number !=1:
        print('%d是素数'%number)
else:
    print('%d不是素数' % number)
'''
练习5：输入两个正整数，计算它们的最大公约数和最小公倍数。
提示：两个数的最大公约数是两个数的公共因子中最大的那个数；两个数的最小公倍数则是能够同时被两个数整除的最小的那个数。
'''
number1=int(input('输入正整数：'))
number2=int(input('输入正整数：'))
if number1 > number2:
    number1,number2=number2,number1
for i in (number1,0,-1):
    if number1 % i ==0 and number2 % i==0:
        print('%d和%d的最大公约数是%d' % (number1, number2, i))
        print('%d和%d的最小公倍数是%d' % (number1, number2, number1 * number2 // i))
        break

'''
练习6：打印如下所示的三角形图案。
*
**
***
****
*****
    *
   **
  ***
 ****
*****
    *
   ***
  *****
 *******
*********
'''
for i in range(0,6):
    for j in range(i+1):
        print('*'*j)

row = int(input('请输入行数: '))
for i in range(row):
    for _ in range(i + 1):
        print('*', end='')
    print()

row = int(input('请输入行数: '))
for i in range(row):
    for j in range(0,row-i-1):
            print(' ',end='')
    for k in range(2*i+1):
            print('*',end='')
    print()


import pygame, sys
import resources.resourceManager as resM
pygame.init()

window = pygame.display.set_mode((200, 200))
background = pygame.Surface((window.get_size()))
background.fill((255, 255, 255))
image = pygame.image.load(resM.heatArrow)
image2 = pygame.image.load(resM.heatArrow)

image = image.convert()
rect = image.get_rect()

image2 = image2.convert_alpha()
rect2 = image2.get_rect()

rect2.left = rect.width + 1

i = 0
while True:
  for event in pygame.event.get():
    if event.type == 12:
      pygame.quit()
      sys.exit()

  image.set_alpha(i)
  image2.set_alpha(i)

  window.fill((255, 255, 255))
  window.blit(background, background.get_rect())
  window.blit(image, rect)
  window.blit(image2, rect2)

  if i == 255:
    i = 0
  else:
    i += 1

  pygame.display.update()
  pygame.time.Clock().tick(60)


import sys
import os
from setuptools import setup

from joseph.constants import REQUIREMENTS_FILE

version = "0.0.1"

def joseph_plugins(requirements=REQUIREMENTS_FILE):
    plugins = []
    if os.path.isfile(requirements):
        with open(requirements, "r") as in_file:
            for line in in_file:
                if not line.startswith(("-", "#")):
                    plugins.append(line)

    return plugins

install_requires = [
    "aiofiles",
    "aiohttp",
    "jinja2",
    "pytimeparse",
    "ruamel.yaml",
    "ruamel.yaml.jinja2",
]

if sys.platform == "darwin":
    # Only require UVLoop on supported systems
    install_requires.append("uvloop")

extra_requires = {
    # Libraries required to run the tests
    'test': [
        "coverage==4.4.1",
        "pytest==3.2.1",
        "mock==2.0.0",
        "tox==2.8.1",
    ],
    # Py.test plugins. Separate to distinguish between test
    # libraries and their plugins
    'pytest.plugins': [
        "pytest-pep8==1.0.6",
        "pytest-cov==2.5.1",
        "pytest-asyncio==0.6.0",
        "pytest-timeout==1.2.0",
    ],
    # Optional Joseph plugins (examples as well as user installed plugins).
    'joseph.plugins': [
        "joseph-hello-world>=0.3.1",
        "joseph_web_hello_world>=0.1.5",
    ] + joseph_plugins()
}

setup(
    name="Joseph_Automation",
    version=version,
    description="A home automation platform with framework aspirations",
    author="Niek Keijzer",
    author_email="info@niekkeijzer.com",
    license="MIT",
    url="https://github.com/NiekKeijzer/joseph",
    packages=["joseph"],
    install_requires=install_requires,
    extras_require=extra_requires,
    test_suite="tests"
)


# -*- coding: UTF-8 -*-
# **********************************************************************************#
#     File:
#   Author: Myron
# **********************************************************************************#
import operator
import numpy as np
from copy import copy
from .. base import *
from ... import logger
from ... core.clock import clock
from ... core.schema import SchemaType
from ... core.enum import SecuritiesType
from ... trade.order import PMSOrder, OrderState, OrderStateMessage
from ... trade.cost import Commission
from ... trade.position import PMSPosition
from ... trade.trade import PMSTrade
from ... utils.dict_utils import DefaultDict, CompositeDict
from ... utils.linked_list_utils import LinkedList, Node
from ... utils.error_utils import Errors
from ... utils.decorator_utils import mutex_lock, scramble_redis_lock
from ... data.database_api import load_adjust_close_price, get_current_st_securities
from ... data.redis_base import redis_queue, RedisCollection

class SecurityBroker(object):

    pool = DefaultDict(LinkedList)
    commission = Commission()
    ST = None
    ST_key = None
    pre_close_price = None
    pre_close_price_key = None
    nodes_positions = DefaultDict()

    def __new__(cls, *args, **kwargs):
        """
        Single instance security broker
        """
        if not hasattr(cls, '_instance'):
            cls._instance = super(SecurityBroker, cls).__new__(cls)
        return cls._instance

    @classmethod
    def get_st_securities(cls):
        """
        Get st securities
        """
        if cls.ST is None or cls.ST_key != clock.current_date:
            cls.ST = get_current_st_securities()
            cls.ST_key = clock.current_date
        return cls.ST

    def get_pre_close_price(self):
        """
        Get pre_close price
        """
        if self.pre_close_price is None or self.pre_close_price_key != clock.current_date:
            self.pre_close_price = load_adjust_close_price(clock.previous_trading_date)
            self.pre_close_price_key = clock.current_date
        return self.pre_close_price

    @scramble_redis_lock()
    def prepare(self):
        """
        load active orders into order pool
        """
        order_today = query_by_securities_('redis', SchemaType.order, clock.current_date,
                                           securities_type=SecuritiesType.SECURITY)
        # active orders group by symbol, submit_time as key
        symbol_orders = CompositeDict()
        for order_schema in order_today.itervalues():
            for order in order_schema.orders.itervalues():
                if not order.order_time:
                    continue
                if order.state \
                        in OrderState.ACTIVE \
                        and order.order_time.split(' ')[0] == clock.current_date.strftime('%Y-%m-%d'):
                    symbol_orders[order.symbol].update({order: order.order_time})
        for orders in symbol_orders.itervalues():
            # order by order time
            order_list = sorted(orders.items(), key=operator.itemgetter(1))
            self.accept_orders([e[0] for e in order_list])

    @mutex_lock
    def accept_orders(self, orders):
        """
        Accept orders
        """
        if isinstance(orders, PMSOrder):
            orders = [orders]
        for order in orders:
            current_order_node = Node(order)
            self.pool[order.symbol].append(current_order_node)

    @mutex_lock
    def transact_minute(self, bar_data):
        """
        Transact for orders by minute bar database
        """
        linked_list = self.pool[bar_data.security_id]
        if not linked_list.get_length():
            return

        # Query chosen positions from redis
        symbol_portfolios = linked_list.recursive(formula=(lambda x, y: x | y),
                                                  formatter=(lambda x: {x.obj.portfolio_id}))
        self.nodes_positions = query_from_('redis', SchemaType.position, portfolio_id=symbol_portfolios)

        # Transact
        volume_ceiling = {e: bar_data.total_volume*100 for e in symbol_portfolios}
        feedback = linked_list.traversal(func=_transact_security_bar, bar_data=bar_data, volume_ceiling=volume_ceiling,
                                         pre_close_price=self.get_pre_close_price(),
                                         nodes_positions=self.nodes_positions)
        changed_orders = map(lambda x: x[0], feedback)
        original_nodes = map(lambda x: x[1], feedback)

        # Update changed orders to redis
        update_('redis', SchemaType.order, changed_orders)

        # del zero amount position
        for position_schema in self.nodes_positions.itervalues():
            if position_schema.positions.get(bar_data.security_id) == 0:
                del position_schema.positions[bar_data.security_id]

        # Dump changed positions to redis
        dump_to_('redis', SchemaType.position, self.nodes_positions)
        self.nodes_positions.clear()

        # Remove inactive orders from linked list
        active_index = [index for index, order in enumerate(changed_orders)
                        if order.state in OrderState.ACTIVE]
        active_orders = list(np.array(changed_orders)[active_index])
        active_nodes = list(np.array(original_nodes)[active_index])
        for node in set(original_nodes) - set(active_nodes):
            linked_list.delete(node)
        linked_list.traversal(func=_synchronize_node, orders=active_orders)

    @classmethod
    def calculate_stock_commission(cls, price, direction):
        """
        Calculate stock commission.

        Args:
            price(float): price
            direction(int): direction
        """
        return cls.commission.calculate_stock_commission(price, direction)

    def post_trading_day(self):
        """
        Post trading days
        """
        self.clear()

    def clear(self):
        """
        Clear info
        """
        self.pool.clear()
        self.nodes_positions.clear()

    @staticmethod
    def playback_daily(order_schema, position_schema, daily_data):
        """
        Transact daily database

        Args:
            order_schema(obj): order schema
            position_schema(obj): position schema
            daily_data(dict): daily database

        Returns:
            (obj, obj): order, position
        """
        return _playback_security_daily(order_schema, position_schema, daily_data)

def _transact_security_bar(node, bar_data, volume_ceiling, pre_close_price, nodes_positions):
    """
    Transact using bar database

    Args:
        node(node): node
        bar_data(database): bar database
    """
    order = copy(node.obj)
    portfolio = nodes_positions.get(order.portfolio_id)
    if portfolio is None:
        _change_order_state(order, OrderState.ERROR, OrderStateMessage.INVALID_PORTFOLIO)
        msg = order.__repr__()
        logger.error('[SECURITY] [TRANSACTION FAILED]'+msg)
        return order, node
    if order.state == OrderState.CANCEL_SUBMITTED:
        _change_order_state(order, OrderState.CANCELED, OrderStateMessage.CANCELED)
        return order, node
    if order.order_amount == 0:
        _change_order_state(order, OrderState.FILLED, OrderStateMessage.FILLED)
        return order, node

    act_price = bar_data.open_price
    pre_close_price = pre_close_price.get(order.symbol) or act_price
    high_price = bar_data.high_price
    low_price = bar_data.low_price
    volume = volume_ceiling.get(order.portfolio_id, 0)
    if volume == 0:
        _change_order_state(order, OrderState.OPEN, OrderStateMessage.NO_AMOUNT)
        return order, node
    limit_pct = 0.0483 if order.symbol in SecurityBroker.get_st_securities() else 0.0993
    if order.direction == 1 and portfolio.cash < act_price * order.order_amount:
        _change_order_state(order, OrderState.REJECTED, OrderStateMessage.NO_ENOUGH_CASH)
        return order, node

    # order.order_type is fixed equal to 'market' currently
    if order.order_type != 'market':
        raise Errors.INVALID_ORDER_TYPE
    commission = order.direction * SecurityBroker.calculate_stock_commission(act_price, order.direction)
    cost_price = act_price + commission
    if order.direction > 0:
        if high_price == low_price and high_price / pre_close_price > 1 + limit_pct:
            _change_order_state(order, target_message=OrderStateMessage.UP_LIMIT)
            return order, node   # 全分钟涨停

        available = \
            min(order.open_amount, volume, portfolio.cash / cost_price) * 1. if cost_price > 0 else 0
        available = int(available) // 100 * 100
    else:
        if order.symbol not in portfolio.positions:
            _change_order_state(order, target_message=OrderStateMessage.SELLOUT)
            return order, node

        if high_price == low_price and high_price / pre_close_price < 1 - limit_pct:
            _change_order_state(order, target_message=OrderStateMessage.DOWN_LIMIT)
            return order, node   # 全分钟跌停

        symbol_available_amount = portfolio.positions[order.symbol].available_amount
        available = min(order.open_amount, volume, symbol_available_amount)
        available = int(available)
        symbol_available_amount -= available

    if available == 0:
        _change_order_state(order, OrderState.OPEN, OrderStateMessage.SELLOUT)
        return order, node

    order._filled_time = ' '.join([portfolio.date, bar_data.bar_minute])
    order._transact_price = (act_price * available + order.transact_price * order.filled_amount) / \
                            (order.filled_amount + available)
    order._filled_amount = order.filled_amount + available
    order._commission = order.commission + abs(available * commission)
    order._turnover_value = order.turnover_value + available * act_price
    pms_trade = PMSTrade(order.order_id, order.symbol, order.direction, order.offset_flag, available,
                         act_price, order.filled_time, commission, None, order.portfolio_id)
    redis_queue.put([pms_trade.to_dict()], key=RedisCollection.trade)
    portfolio.cash -= order.direction * cost_price * available
    if order.filled_amount == order.order_amount:
        _change_order_state(order, OrderState.FILLED, OrderStateMessage.FILLED)
    elif order.filled_amount == 0:
        _change_order_state(order, OrderState.OPEN)
    else:
        _change_order_state(order, OrderState.PARTIAL_FILLED)

    volume_after = max((volume - available), 0)
    volume_ceiling.update({order.portfolio_id: volume_after})
    position = portfolio.positions.get(order.symbol)
    pre_amount = 0 if position is None else position.amount
    if pre_amount:
        position.amount += order.direction * available
        if order.direction == -1:
            position.total_cost = position.total_cost * position.amount / pre_amount if position.amount != 0 else 0
            position.available_amount -= available
        else:
            position.total_cost = position.total_cost + order.direction * available * cost_price
        position.cost = position.total_cost / position.amount if position.amount else 0
    else:
        if position is None:
            position = PMSPosition(order.symbol, 0, 0, 0.0)
            portfolio.positions[order.symbol] = position
        position.amount = order.direction * available
        position.total_cost = order.turnover_value + order.commission
        position.cost = position.total_cost / position.amount
    position.value = act_price * position.amount
    msg = order.__repr__()
    logger.debug('[SECURITY] [TRANSACTION]'+msg)
    return order, node

def _playback_security_daily(order_schema, position_schema, daily_data):
    """
    Transact securities by daily database

    Args:
        order_schema(obj): order schema
        position_schema(obj): position schema
        daily_data(dict): daily database

    Returns:
        (obj, obj): order, position
    """
    # todo. transact order_schema based on position_schema
    playback_order_schema, playback_position_schema = order_schema, position_schema
    return playback_order_schema, playback_position_schema

def _synchronize_node(node, orders):
    """
    Synchronize orders to linked list

    Args:
        node(node): node
        orders(list): orders
    """
    node.obj = orders.pop(0)

def _change_order_state(order, target_state=None, target_message=None):
    """
    更新当前order状态，暂不处理非active的Order

    Args:
        order (PMSOrder): 需要更新的order
        target_state (str): 目标状态
        target_message (str): 目标状态提示

    """
    if order is None:
        return
    if not isinstance(order, PMSOrder):
        raise Errors.INVALID_ORDER_OBJECT
    if target_state is not None:
        order._state = target_state
    if target_message is not None:
        order._state_message = target_message


#!/usr/local/bin/python
import errno
import numpy as np
import os
import sys
import json
# matplotlib.rcParams['figure.figsize'] = [3.125, 1.93]
marker = [".", "o", "x", "s", "*"]
#algos = ["ranking"]
#algos = ["p1", "p2", "p2+p3", "p2+p3+p4", "p2+p3+p4+p5"]
#algos = ["homa", "ndp", "hpcc", "pim"]
algos = ['pim']
#algos = ["phost"]
#traces = ["aditya"]
traces = ["worstcase2"]
# traces = ['aditya', 'dctcp']
# input_file1 = sys.argv[1]
# output_file = sys.argv[2]
load = 0.8

ID = 0
SIZE = 1
START_TIME = 2
END_TIME = 3
FCT = 4
ORCT = 5

def read_file(filename):
    output = []
    total_sent_packets = 0
    total_packets = 0
    finish_time = 0
    s_time = 1.0
    reach_check_point = 0
    with open(filename) as f:
        lines = f.readlines()
        for i in range(len(lines) - 1):
            line = lines[i]
            params = line.split()
            output.append([params[0], float(params[1]) / 1600,])
    return output

def output_file(output, filename):
    workload = ""
    file = open(filename, "w+")
    i = 0
    for line in output:
        string = ""
        string += line[0] + " " + str(line[1])
        string += "\n"
        file.write(string)
        i += 1
        if i > 60:
            break

def main():
    output =  read_file("result/ndp/worst_case/thpt_NDP_worstcase1.dat" )
    # draw_graph(util, trace + " Utilization")
    # draw_graph(fct_oct_ratio, trace + " Slowdown")
    # print util, fct_oct_ratio, fct_oct_ratio_99
    # output_file(util, "../gnuplot/data/{0}_util.dat".format(trace))
    output_file(output, "result/ndp/worst_case/util_NDP_worstcase1.txt")
    # output_file(fct_oct_ratio_99, "../gnuplot/data/{0}_99_slowdown.dat".format(trace))

main()


import sys
import random

random.seed()

urllib = []
for i in range(10):
    urllib.append('url' + str(i))
i = 0

if len(sys.argv) == 1:
    max = 10
else:
    max = (int(sys.argv[1]))

for file_name in ['t1.txt', 't2.txt', 't3.txt']:
    f = open(file_name, 'w')
    count = random.randrange(int(max/1.5), max)
    urls = random.sample(urllib[0:max-1:], count)
    for i in range(count):
        f.write(urls[i] + '\n')
    f.close()


# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

""" Copyright 2011 GPL licence applies"""

bl_info = {
    "name": "Advanced UI Menus",
    "description": "Menus for advanced interaction with blender's UI",
    "author": "Ryan Inch",
    "version": (1, 1),
    "blender": (2, 72),
    "location": "View3D - Multiple menus in multiple modes.",
    "warning": '',  # used for warning icon and text in addons panel
    "wiki_url": "http://wiki.blender.org/index.php/Extensions:2.6/Py/Scripts/3D_interaction/Advanced_UI_Menus",
    "category": "3D View"}

import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'advanced_ui_menus'))

from core import *

import brush_options
import brushes
import curve_menu
import delete_menu
import dyntopo_menu
import extrude_menu
import manipulator_menu
import mode_menu
import paint_control
import operators
import pivot_menu
import proportional_menu
import custom_menu
import selection_menu
import shade_menu
import snap_menu
import stroke_options
import symmetry_options
import texture_options
import view_menu

addon_files = [brushes,
              brush_options, curve_menu,
              delete_menu, dyntopo_menu,
              extrude_menu, manipulator_menu,
              mode_menu, paint_control,
              operators, pivot_menu,
              proportional_menu, custom_menu,
              selection_menu, shade_menu,
              snap_menu, stroke_options,
              symmetry_options, texture_options,
              view_menu]

# this is a list of keys to deactivate with a list of modes to deactivate them in
keymodes = [  #    modes            key    any   shift  ctrl   alt   oskey
            [['Object Non-modal'], 'TAB', False, False, True, False, False],

            [['Mesh'], 'TAB', False, False, True, False, False],

            [['Object Non-modal'], 'TAB', False, False, False, False, False],

            [['Object Mode', 'Object Non-modal','Mesh','Curve','Metaball','Lattice','Particle'], 'O', False, False,
              False, False, False],

            [['Object Mode', 'Object Non-modal','Mesh','Curve','Metaball','Lattice','Particle'], 'O', False, True,
              False, False, False],

            [['Mesh'], 'X', False, False, False, False, False],

            [['Object Non-modal', 'Weight Paint'], 'V', False, False, False, False, False],

            [['3D View'], 'TAB', False, True, False, False, False],

            [['3D View'], 'TAB', False, True, True, False, False],
            [['Mesh'], 'E', False, False, False, False, False]
            ]

blender_on = False

def register():
    global blender_on

    # register all files
    for addon_file in addon_files:
            addon_file.register()

    # start a thread for disabling the opposing keymaps if this is blenders first start
    if not blender_on:
        thread = Thread(target=opposingkeys, args=(keymodes, False, True))
        thread.start()

    # else just disable the opposing keymaps
    else:
        opposingkeys(keymodes, False)

    # mark blender as on
    blender_on = True

def unregister():

    # unregister all files
    for addon_file in addon_files:
        addon_file.unregister()

    # re-enable all the keymaps you disabled
    opposingkeys(keymodes, True)

    # delete all the properties you have created
    del_props()

if __name__ == "__main__":
    register()


### This is a template file meant to be a guideline to smooth out the implementation of our models in the same framework.
##	William Korcari: william.korcari@desy.de

## import of the models
from simple_graph_net import Network

#########################################

nn = Network()

datasets = nn.compatible_datasets

for ds in datasets:

    x_train, y_train = ds.load_graph('train', path = './datasets')
    x_test, y_test = ds.load_graph('test', path = './datasets')

    model = nn.model(ds, shapes=nn.get_shapes(x_train))
    model.compile(**nn.compile_args(ds.task))
    print(model.summary())
    history = model.fit(x=x_train, y=y_train, **nn.fit_args)

    # after training saving best model
    filepath = './trained_models/{}/{}_model'.format(ds.name, nn.model_tag(ds.name, nn.model_name))
    model.save(filepath)

    # evaluation after training
    nn.evaluation(
        model=model,
        history=history,
        dataset=ds,
        x_test=x_test,
        y_test=y_test,
        model_name = nn.model_name,
        path = filepath,
    )


# https://github.com/DavidWangWood/Camera-Calibration-Python/blob/master/camera_calibration.ipynb
# https://docs.opencv.org/3.3.0/dc/dbb/tutorial_py_calibration.html
# https://docs.opencv.org/master/dc/dbb/tutorial_py_calibration.html

import numpy as np
import cv2
import glob

# termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((7 * 10, 3), np.float32)
objp[:, :2] = np.mgrid[0:10, 0:7].T.reshape(-1, 2)

# Arrays to store object points and image points from all the images.
objpoints = []  # 3d point in real world space
imgpoints = []  # 2d points in image plane.
# images = glob.glob('*.bmp')
images=glob.glob('*.bmp')
print(images)
k=0
p=0
for fname in images:
    img = cv2.imread(fname)
    img_size = (img.shape[1], img.shape[0])
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Find the chess board corners
    ret, corners = cv2.findChessboardCorners(gray, (10, 7), None)
    if ret==False:
        p+=1
    print("ominięto", p)
    # If found, add object points, image points (after refining them)
    if ret == True:
        k+=1
        objpoints.append(objp)
        corners2 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners)

        # Draw and display the corners

        kozak=cv2.drawChessboardCorners(gray, (10, 7), corners2, ret)
        cv2.imwrite(f"obrazek{k}.jpg",kozak)
        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)
        k+=1
        print(ret)
        print("przyjeto",k)
        dst = cv2.undistort(img, mtx, dist, None, mtx)

        cv2.imwrite(f'obrazek{k}.jpg', dst)
        # dst = cv2.undistort(dist, mtx, dist, None, mtx)
        # cv2.imshow('img',gray)

        cv2.waitKey(0)

# cv2.destroyAllWindows()
img=cv2.imread('img00308_r.bmp')
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)
print(ret)
dst = cv2.undistort(img, mtx, dist, None, mtx)
cv2.imwrite('undistorted.jpg', dst)

import matplotlib.pyplot as plt

# Test undistortion on an image

# Do camera calibration given object points and image points
# ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)
#
# dst = cv2.undistort(img, mtx, dist, None, mtx)
# cv2.imwrite('TEST.jpg', dst)
#
#
# # dst = cv2.cvtColor(dst, cv2.COLOR_BGR2RGB)
# # Visualize undistortion
# f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
# ax1.imshow(img)
# ax1.set_title('Original Image', fontsize=30)
# ax2.imshow(dst)
# ax2.set_title('Undistorted Image', fontsize=30)
# plt.show()


import matplotlib.pyplot as plt
import numpy as np
from math import sqrt
K =23.26
r = 12/4000
tau = 0.51
Kampli = 10
print(-(1/(4*tau**2))+(K*r*Kampli)/(tau))
A = sqrt(-(1/(4*tau**2))+(K*r*Kampli)/(tau))

x=np.linspace(0,10,1000)
p1 = plt.plot(x,
            (
                1
                -np.exp(-x/(2*tau))*np.cos(A*x)
                -(1)/(2*tau*A)*np.exp(-x/(2*tau))*np.sin(A*x)
                ),label="Angle de sortie $\\theta(t)$",lw=2)
p2 = plt.plot(x,
            (
                1+0.*x
                ),label="Angle d'entrée $\\theta_e(t)$",lw=2)

plt.grid(True, which="both", linestyle="dotted")
#plt.title("Fonctions trigonometriques")
#plt.legend([p1, p2], ["Sinus", "Cosinus"])
plt.legend(loc='lower right', fancybox=True, shadow=True, prop=dict(size=10))
plt.xlabel("Temps en $s$")
plt.ylabel("Angle en $rad$")
#plt.title("Evolution de $\theta$ en fonction du temps pour une entrée échelon de 1V")
plt.show()
fig = plt.figure()
#fig.savefig('temp.pdf', transparent=True)


import drawSvg as draw
import numpy as np
import random as rnd
import hashlib
import time
import webcolors

t = str(time.time())
s = t.encode()
hash_object = hashlib.sha256(s)
seed = hash_object.hexdigest()
print(seed)

rnd.seed(seed,version=2)

########### parameters##########################################################

HEIGHT = 800
WIDTH = 600

divider = 5

SW = [0.5,0.5,1,1,1,1,1,2,2,2,2,3,3]

nLayers = rnd.randint(3,10)
layers = np.round(np.linspace(0,HEIGHT,nLayers)).astype(int)

points_per_layer = np.round((np.random.random(nLayers)*10) + 1).astype(int)

points_per_layer[::2] = points_per_layer[::2]*5

points = []
for i in range(nLayers):
    row = np.arange(0,WIDTH)
    this_row = np.sort(np.random.choice(row,points_per_layer[i]))
    points.append(this_row)

# print(points_per_layer)
# print(layers)
# print(points)

########### drawing ############################################################
d = draw.Drawing(WIDTH,HEIGHT)

d.append(draw.Line(0,0, WIDTH,0, stroke_width=int(divider*2), stroke='black'))
d.append(draw.Line(0,HEIGHT, WIDTH,HEIGHT, stroke_width=int(divider*2), stroke='black'))

for r in range(len(points)-1):
    L = draw.Line(0,layers[r], WIDTH,layers[r], stroke_width=divider, stroke='black')
    d.append(L)

    for p in range(len(points[r])):
        sw = rnd.choice(SW)
        adder = rnd.randint(0,5)
        line = draw.Line(points[r][p],layers[r], points[r+1][np.mod(p+adder,len(points[r+1]))],layers[r+1], stroke_width=sw, stroke='black')

        d.append(line)

# d.append(draw.Line(thispoint[0],thispoint[1], newpt[0],newpt[1], stroke_width=sw, stroke='black'))

d.saveSvg('bipart.svg')


from base import *
import dj_database_url

WSGI_APPLICATION = 'gettingstarted.wsgi.application'

# Update database configuration with $DATABASE_URL.
db_from_env = dj_database_url.config(conn_max_age=500)
DATABASES['default'].update(db_from_env)
DATABASES = {'default': dj_database_url.config(default='postgres://localhost')}

DEBUG = False


import os, sys, argparse, re, glob
from utils import *

parser = argparse.ArgumentParser()
parser.add_argument('--numRuns', type=int, help='Define number of runs to be analysed.', required=True)
parser.add_argument('--hepd', action='store_true', help='Analyse HEPD data.')
parser.add_argument('--hepp', action='store_true', help='Analyse HEPP data.')
parser.add_argument('--merge', action='store_true', help='Merge all runs.')
parser.add_argument('--ana', action='store_true', help='Analyse all runs.')
parser.add_argument('-q','--quiet', action='store_true', help='Run without printouts.')
args,_=parser.parse_known_args()

runs = args.numRuns
os.system('source /opt/exp_software/limadou/set_env_standalone.sh')

datapaths = []
if args.hepd:
    datapaths = glob.glob('/storage/gpfs_data/limadou/data/flight_data/L3h5/*.h5')
# select HEPP data from 22-26.02.2019 (solar quiet period)
elif args.hepp:
    datapaths = glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190222*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190223*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190224*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190225*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190226*.h5')

if not args.merge and not args.ana:

    if len(datapaths) < runs:
        print("Only {} files available for reading. ".format(len(datapaths)))
        runs = len(datapaths)

    for irun,run in enumerate(datapaths):
        if irun>(runs-1):
            break
        outfile = home()+"/root/"+(os.path.split(run)[1]).replace("h5","root")

        print("Test if output exists: ", outfile)
        if os.path.isfile(outfile):
            print("Output root file already exists... \n read in the next file. ")
            runs=runs+1
        else:
            cmd='python3 python/readH5.py --inputFile '+str(run)
            if not args.quiet:
                cmd+=' --debug'
            if args.hepd:
                cmd+=' --data hepd'
            elif args.hepp:
                cmd+=' --data hepp'
            print(cmd)
            os.system(cmd)

# run analysis on single merged root file
elif args.ana:
    mge = 'root/all_hepd_'+str(runs)+'runs.root'
    print("Run analysis on single file: ", str(mge))
    if args.hepp:
        mge = 'root/all_hepp_'+str(runs)+'runs.root'

    # open root file
    runana='python3 python/analyseRoot.py --inputFile '+str(mge)
    os.system(runana)

# merge all root files
elif args.merge:
    mge=''
    runList=[]

    if args.hepd:
        # write
        mge = 'root/all_hepd_'+str(runs)+'runs.root'
        runList = glob.glob('root/CSES_HEP_DDD_*.root')
    elif args.hepp:
        mge = 'root/all_hepp_'+str(runs)+'runs.root'
        runList = glob.glob('root/CSES_01_HEP_1_*.root')

    if len(runList) < runs:
        print("Only {} files available for merge. ".format(len(runList)))
        runs = len(runList)
        if args.hepd:
            mge = 'root/all_hepd_'+str(runs)+'runs.root'
        elif args.hepp:
            mge = 'root/all_hepp_'+str(runs)+'runs.root'

    print("Merge files in: ", mge)
    merge(mge, runList, runs)


import os
import sys
import ConfigParser
from nginxparser import load, dumps

from pprint import pprint

def init_config(filename="/etc/php5/fpm/pool.d/www.conf"):
    Config = ConfigParser.ConfigParser()
    Config.read(filename)
    return Config

def rename_section(filename="/etc/php5/fpm/pool.d/www.conf", existing_name="www", new_name=None):
    """
    Renames a configuraiton section.

    Does so by actually copying all config items to a new section, then removing the original section.

    :param filename: the path to the file to be editied
    :param existing_name: the existing section name
    :param new_name: the new section name
    """
    Config = ConfigParser.ConfigParser()
    Config.read(filename)
    section_items = Config.items(existing_name)
    for item in section_items:
        _opt_move(Config
                  , section1=existing_name
                  , section2=new_name
                  , option=item)
    Config.remove_section(existing_name)

    with open(filename, 'wb') as configfile:
        Config.write(configfile)

def _opt_move(config, section1, section2, option):
    """
    Helper function for moving an existing option from one section to another.

    :type section2: str
    :type section1: str
    :type config: object
    :type option: tuple
    :param config: an config object
    :param section1: name of the original section
    :param section2: name of the new section
    :param option: a tuple of the configuration option (name,value)

    """
    try:
        config.set(section2, option[0], config.get(section1, option[0]))
    except ConfigParser.NoSectionError:
        # Create non-existent section
        config.add_section(section2)
        _opt_move(config, section1, section2, option)
    else:
        config.remove_option(section1, option[0])

def envs_to_configs(filename="/etc/php5/fpm/pool.d/www.conf", sectionname="www", prefix="wp"):
    """
    Converts environment variables into php-fpm variables.

    Writes or updates them to the specified file (should be a pool file).
    :type prefix: str
    :type sectionname: str
    :type filename: str
    :param filename: The filename to write to.
    :param sectionname: The sectionname to write to.
    :param prefix: the prefix to filter environment variables.
    """

    # env[MY_VAR]
    variable_list = get_vars(prefix=prefix)
    theconfig = init_config(filename=filename)
    for variable in variable_list:
        thename = "env[{0}]".format(variable.get("key"))
        thevalue = variable.get("val")
        output = "env[{0}] = {1}".format(thename, thevalue)
        print(output)
        write_to_conf(theconfig, sectionname=sectionname, option=thename, value=thevalue)

    with open(filename, 'wb') as configfile:
        theconfig.write(configfile)

def write_to_conf(configObj, sectionname="www", option=None, value=None):
    Config = configObj
    try:
        Config.set(section=sectionname, option=option, value=value)
    except ConfigParser.NoSectionError as e:
        Config.add_section(sectionname)

def get_vars(varname=None, prefix=None):
    """
    Retrieves all environment variables on the system and returns them as a list of dictionaries.

    :type prefix: str
    :rtype: list
    :param prefix: THe prefix to filter for.
    :return: A list of environment variables which are a dictionaries with "name" and "value" keys.
    """
    if varname:
        thevar = os.environ.get(varname)
        return thevar
    variables_to_show = sorted(os.environ.keys())
    envarlist = list()
    for index, variable in enumerate(variables_to_show):
        if prefix in variable:
            value = os.environ.get(variable)
            envvar = dict(
                key=variable,
                val=value
            )
            envarlist.append(envvar)
    return envarlist

def turnon_pagespeed():
    found_setting = get_vars(varname="enable_pagespeed")
    if found_setting and found_setting in ("on","off"):
        print("--------Found environment setting for pagespeed, requested to set pagespeed " + found_setting+ "! --------")
        ps_config = load(open("/etc/nginx/conf.d/pagespeed.conf"))
        #print(ps_config.index(["pagespeed", "off"]))
        for idx, setting in enumerate(ps_config):
            if len(setting[1].split(" ")) == 1 and setting[1] in ("on","off"):
             current_setting = setting[1]
             print("--------Found the on/off setting at index "+ str(idx)+" --------")
             print("--------Currently pagespeed is set to "+ current_setting+". ---------")
             del ps_config[idx]
             ps_config.insert(0,["pagespeed", found_setting])
        outfile = open("/etc/nginx/conf.d/pagespeed.conf","w")
        outfile.write(dumps(ps_config))
        outfile.close()
        print("-------Pagespeed is now set to "+found_setting+"-------")

    else:
        print("No environment variable found, pagespeed will remain disabled.")

def do_config(appname=None):
    """
    Sets up wp the php-fpm environment.

    Requestes one commandline parameters, env_prefix.
    * env_prefix is the prefix of enviroment variables that we want to convert to php-fpm env varialbes.

    The app name is set from an environment variable called "app_name", which should be set when the container is run.
    This will change the name of the fpm process to be the name of the app so that it is easier to track in ps and top.

    :param appname:
    """
    app_name = get_vars(varname="app_name")
    arguments = sys.argv
    if len(arguments) != 2:
        raise AttributeError("You must specify --env_prefix")
        sys.exit(status=1)
    parsedoptlist = list()
    for arg in arguments:
        if "--" in arg:
            full = arg.split("--")[1].split("=")
            option = full[0]
            thevalue = full[1]
            parsedopt = (option, thevalue)
            if option in ("env_prefix"):
                parsedoptlist.append(parsedopt)
    if len(parsedoptlist) != 1:
        raise AttributeError("You must specify --env_prefix")
        sys.exit(1)
    for arg in parsedoptlist:
        if arg[0] == "env_prefix":
            env_prefix = arg[1]
    print("-------- Setting up the environment variables using the variable prefix: "+env_prefix+". -------")
    envs_to_configs(prefix=env_prefix)
    if app_name:
        print("-------- Setting the app name to: "+app_name+". --------")
        rename_section(new_name=app_name)
    else:
        print("No app name was set, leaving the default app name.")
    # Now we do pagespeed on or off setting
    turnon_pagespeed()

if __name__ == "__main__":
    do_config()


from flask import Flask,render_template,redirect,url_for,session, request, jsonify, make_response
from service.controller import bp_user as app
#디비
from service.model import dbHelper #객체 변수 땡겨옴
from service import config

#라우팅
@app.route('/')
def home() :
    #세션이 없으면 /login으로 리다이렉트
    # 만약 이 서버에 접속하는 사람이 2명이상일때도 이 조건문이 성립하는가?
    # if not 'uid' in session :
    #     return redirect(url_for('login'))

    ################################################
    # 쿠키 적용 : 응답을 이용 - 사용자의 브라우저에다가 특정정보를 남기는 것
    # ->아이디를 저장해서 로그인페이지 뜰때 자동으로 아이디가 보이게 하기.
    # 응답 객체를 생성한다.

    # 쿠키 세팅
    resp=make_response(render_template('index.html',config=config)) #render template 반환할때 쿠키 세팅해서 응답하기.
    resp.set_cookie('uid', session['uid'])
    return resp

#~/url/login ->url_for로 구현되기때문에 굳이 이렇게 url이 바뀌어도 괜찮게 됨..!!!
@app.route('/login',methods=['POST','GET'])
def login():
    if request.method=='GET':
        #쿠키 획득 : get방식일때
        uid=request.cookies.get('uid')
        #쿠키가 없을 경우 : None으로 나오기 때문에 기본값 처리
        if not uid: #uid==None
            uid=''
        config.cookie_uid=uid
        return render_template('login.html',config=config)
    else :
        uid=request.form['uid'] #post 방식으로 요청
        upw=request.form['upw']
        row=dbHelper.loginSql(uid=uid,upw=upw) #디비쿼리수행
        # false :[],(),{},0,""
        # row ->dit =>{..}
        # row에 uid upw name 저장
        if row :
            #세션처리(필요한 정보를 세션으로 저장한다) -서버 메모리에 저장
            session['uid']= uid
            session['name']=row['name']
            #세션종료
            return redirect(url_for('userbp.home'))
        else :
            return render_template('common/alert.2.html',msg='회원이 아닙니다')

    #경고창 ->되돌아가기

#로그아웃
#~/user/logout
@app.route('/logout')
def logout():
    if not 'uid' in session :
        return redirect(url_for('userbp.login'))
    #세션 종료
    print(session)
    if 'uid' in session:
        session.pop('uid',None)
    if 'name' in session:
        session.pop('name',None)
        print("*"*50)
    print('세션제거후 :',session)
    #페이지 요청을 리다이렉트 ->홈페이지
    return redirect(url_for('userbp.home'))


import csv
import random
import os

def split_list(files, train_rate=0.8):
    random.shuffle(files)
    m = int(len(files) * train_rate)
    train_data = files[:m]
    test_data = files[m:]

    return train_data, test_data

def writte_csv(csv_file, rows):
    for i in rows:
        csv_file.writerow(i)

CSV_FILE = "train.csv"
CSV_MALE_TRAIN = "male_train.csv"
CSV_FEMALE_TRAIN = "female_train.csv"
CSV_MALE_TEST = "male_test.csv"
CSV_FEMALE_TEST = "female_test.csv"
CSV_UNISEX_TRAIN = "unisex_train.csv"
CSV_UNISEX_TEST = "unisex_test.csv"
PATH_OF_FILES = "../datasets/dnlm_train"

files = os.listdir(PATH_OF_FILES)
files.sort()

id = 'id'
boneage = 'boneage'
male = 'male'
fieldnames = [id, boneage]

males = []
females = []
unisex = []

csvfile = open(CSV_FILE)
reader = csv.DictReader(csvfile)

for row in reader:
    temp =  row[id] + ".png"
    if temp in files:
        if row[male] == 'True':
            males.append({id:row[id], boneage:row[boneage]})
            #writtermale.writerow({id:row[id], boneage:row[boneage]})
        else:
            #writtefemale.writerow({id:row[id], boneage:row[boneage]})
            females.append({id:row[id], boneage:row[boneage]})
        unisex.append({id:row[id], boneage:row[boneage]})

# open male files
csv_male_train = open(CSV_MALE_TRAIN, 'w')
csv_male_test = open(CSV_MALE_TEST, 'w')

# open female files
csv_female_train = open(CSV_FEMALE_TRAIN, 'w')
csv_female_test = open(CSV_FEMALE_TEST, 'w')

# open unisex files
csv_unisex_train = open(CSV_UNISEX_TRAIN, 'w')
csv_unisex_test = open(CSV_UNISEX_TEST, 'w')

# writters de cada csv respectivo
writter_male_train = csv.DictWriter(csv_male_train, fieldnames=fieldnames)
writter_male_test = csv.DictWriter(csv_male_test, fieldnames=fieldnames)

writter_female_train = csv.DictWriter(csv_female_train, fieldnames=fieldnames)
writter_female_test = csv.DictWriter(csv_female_test, fieldnames=fieldnames)

writter_unisex_train = csv.DictWriter(csv_unisex_train, fieldnames=fieldnames)
writter_unisex_test = csv.DictWriter(csv_unisex_test, fieldnames=fieldnames)

# writte headers
writter_male_train.writeheader()
writter_male_test.writeheader()
writter_female_train.writeheader()
writter_female_test.writeheader()
writter_unisex_train.writeheader()
writter_unisex_test.writeheader()

# splitea 80~20 a male
male_train, male_test = split_list(males)

# crea los csv de cada dataset
writte_csv(writter_male_train, male_train)
writte_csv(writter_male_test, male_test)

# splitea 80~20 a female
female_train, female_test = split_list(females)

# crea los csv de cada dataset
writte_csv(writter_female_train, female_train)
writte_csv(writter_female_test, female_test)

# splitea 80~20 a unisex
unisex_train, unisex_test = split_list(unisex)

# crea los csv de cada dataset
writte_csv(writter_unisex_train, unisex_train)
writte_csv(writter_unisex_test, unisex_test)

csvfile.close()
csv_male_train.close()
csv_male_test.close()
csv_female_train.close()
csv_female_test.close()
csv_unisex_train.close()
csv_unisex_test.close()


from django.urls import include, path
from rest_framework.authtoken import views
from rest_framework.routers import DefaultRouter

from .views import CommentViewSet, ReviewViewSet

router = DefaultRouter()

router.register("titles/(?P<title_id>\d+)/reviews", ReviewViewSet, basename="reviews")
router.register(
    "titles/(?P<title_id>\d+)/reviews/(?P<review_id>\d+)/comments",
    CommentViewSet,
    basename="comments",
)

urlpatterns = [
    path("api/v1/", include(router.urls)),
]


#
# Collective Knowledge (individual environment - setup)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#

import os

##############################################################################
# limit directories

def limit(i):

    dr=i.get('list',[])
    drx=[]

    for q in dr:
        if q.find('.libs')<0 and q.find('anaconda')<0:
           drx.append(q)

    # Anaconda installation and libraries often conflict with existing OS installations

    return {'return':0, 'list':drx}

##############################################################################
# get version from path

def version_cmd(i):

    ck=i['ck_kernel']

    fp=i['full_path']

    ver=''

    r = ck.access({'action': 'search_version',
                   'module_uoa': 'soft',
                   'path': fp})
    if r['return']>0: return r
    ver=r.get('version','')

    if ver=='':
       if len(fp)>0:
          j=fp.rfind('.')
          if j>0:
             fps=fp[:j]+'.settings'
             if os.path.isfile(fps):
                # Load file and find setting
                r=ck.load_text_file({'text_file':fps, 'split_to_list':'yes'})
                if r['return']>0: return r

                for l in r['lst']:
                    l=l.strip()
                    if l.startswith('HDF5 Version:'):
                       ver=l[14:].strip()
                       break

    return {'return':0, 'cmd':'', 'version':ver}

##############################################################################
# setup environment setup

def setup(i):
    """
    Input:  {
              cfg              - meta of this soft entry
              self_cfg         - meta of module soft
              ck_kernel        - import CK kernel module (to reuse functions)

              host_os_uoa      - host OS UOA
              host_os_uid      - host OS UID
              host_os_dict     - host OS meta

              target_os_uoa    - target OS UOA
              target_os_uid    - target OS UID
              target_os_dict   - target OS meta

              target_device_id - target device ID (if via ADB)

              tags             - list of tags used to search this entry

              env              - updated environment vars from meta
              customize        - updated customize vars from meta

              deps             - resolved dependencies for this soft

              interactive      - if 'yes', can ask questions, otherwise quiet
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0

              bat          - prepared string for bat file
            }

    """

    import os

    # Get variables
    ck=i['ck_kernel']
    s=''

    iv=i.get('interactive','')

    cus=i.get('customize',{})
    fp=cus.get('full_path','')

    hosd=i['host_os_dict']
    tosd=i['target_os_dict']

    win=tosd.get('windows_base','')
    winh=hosd.get('windows_base','')

    # Check platform
    hplat=hosd.get('ck_name','')
    tplat=tosd.get('ck_name','')

    hproc=hosd.get('processor','')
    tproc=tosd.get('processor','')

    remote=tosd.get('remote','')
    tbits=tosd.get('bits','')

    env=i['env']
    ep=cus['env_prefix']

    fi=cus.get('include_file','')

    pinc=fp
    fpinc=''
    found=False
    while True:
       fpinc=os.path.join(pinc,'include', fi)
       if os.path.isfile(fpinc):
          found=True
          break
       fpinc=os.path.join(pinc,'include','hdf5', 'serial', fi)
       if os.path.isfile(fpinc):
          found=True
          break
       pincx=os.path.dirname(pinc)
       if pincx==pinc:
          break
       pinc=pincx

    if not found:
       return {'return':1, 'error':'can\'t find include directory for HDF5'}

    pi=os.path.realpath(os.path.dirname(fpinc))

    pii=os.path.dirname(pi)

    lb=os.path.basename(fp)
    lbs=lb
    if lbs.endswith('.so'):
       lbs=lbs[:-3]+'.a'
    elif lbs.endswith('.lib'):
       lbs=lbs[:-4]+'.dll'

    pl=os.path.realpath(os.path.dirname(fp))
    cus['path_lib']=pl

    pl1=os.path.dirname(pl)
    pl2=os.path.dirname(pl1)

    cus['path_include']=pi
    cus['include_name']=fi

    cus['static_lib']=lb
    cus['dynamic_lib']=lbs

    r = ck.access({'action': 'lib_path_export_script', 'module_uoa': 'os', 'host_os_dict': hosd,
      'lib_path': cus.get('path_lib','')})
    if r['return']>0: return r
    s += r['script']

    env[ep]=pii

    pb=os.path.join(pii,'bin')
    if os.path.isdir(pb):
       env[ep+'_BIN']=pb
       cus['path_bin']=pb
       if tplat=='win':
          s+='\nset PATH='+pb+';%PATH%\n\n'

          env[ep+'_LFLAG']=os.path.join(pl,'hdf5.lib')
          env[ep+'_LFLAG_HL']=os.path.join(pl,'hdf5_hl.lib')

    env[ep+'_INCLUDE_NAME']=cus.get('include_name','')
    env[ep+'_STATIC_NAME']=cus.get('static_lib','')
    env[ep+'_DYNAMIC_NAME']=cus.get('dynamic_lib','')

    return {'return':0, 'bat':s}


#! /usr/bin/env python

"""Multilayer Perceptron for drug response problem"""

from __future__ import division, print_function

import argparse
import csv
import logging
import sys

import numpy as np
import pandas as pd

from itertools import tee, islice

from keras import backend as K
from keras import metrics
from keras.models import Sequential
from keras.layers import Dense, Dropout, LocallyConnected1D, Convolution1D, MaxPooling1D, Flatten
from keras.callbacks import Callback, ModelCheckpoint, ProgbarLogger

from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler

# For non-interactive plotting
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

import p1b3

# Model and Training parameters

# Seed for random generation
SEED = 2016
# Size of batch for training
BATCH_SIZE = 100
# Number of training epochs
NB_EPOCH = 20
# Number of data generator workers
NB_WORKER = 1

# Percentage of dropout used in training
DROP = 0.1
# Activation function (options: 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear')
ACTIVATION = 'relu'
LOSS = 'mse'
OPTIMIZER = 'sgd'
# OPTIMIZER = 'adam'

# Type of feature scaling (options: 'maxabs': to [-1,1]
#                                   'minmax': to [0,1]
#                                   None    : standard normalization
SCALING = 'std'
# Features to (randomly) sample from cell lines or drug descriptors
FEATURE_SUBSAMPLE = 500#0
# FEATURE_SUBSAMPLE = 0

# Number of units in fully connected (dense) layers
D1 = 1000
D2 = 500
D3 = 100
D4 = 50
DENSE_LAYERS = [D1, D2, D3, D4]

# Number of units per locally connected layer
C1 = 10, 10, 5       # nb_filter, filter_length, stride
C2 = 0, 0, 0         # disabled layer
# CONVOLUTION_LAYERS = list(C1 + C2)
CONVOLUTION_LAYERS = [0, 0, 0]
POOL = 10

MIN_LOGCONC = -5.
MAX_LOGCONC = -4.

CATEGORY_CUTOFFS = [0.]

np.set_printoptions(threshold=np.nan)
np.random.seed(SEED)

def get_parser():
    parser = argparse.ArgumentParser(prog='p1b3_baseline',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="increase output verbosity")
    parser.add_argument("-a", "--activation", action="store",
                        default=ACTIVATION,
                        help="keras activation function to use in inner layers: relu, tanh, sigmoid...")
    parser.add_argument("-b", "--batch_size", action="store",
                        default=BATCH_SIZE, type=int,
                        help="batch size")
    parser.add_argument("-c", "--convolution", action="store", nargs='+', type=int,
                        default=CONVOLUTION_LAYERS,
                        help="integer array describing convolution layers: conv1_nb_filter, conv1_filter_len, conv1_stride, conv2_nb_filter, conv2_filter_len, conv2_stride ...")
    parser.add_argument("-d", "--dense", action="store", nargs='+', type=int,
                        default=DENSE_LAYERS,
                        help="number of units in fully connected layers in an integer array")
    parser.add_argument("-e", "--epochs", action="store",
                        default=NB_EPOCH, type=int,
                        help="number of training epochs")
    parser.add_argument("-l", "--locally_connected", action="store_true",
                        default=False,
                        help="use locally connected layers instead of convolution layers")
    parser.add_argument("-o", "--optimizer", action="store",
                        default=OPTIMIZER,
                        help="keras optimizer to use: sgd, rmsprop, ...")
    parser.add_argument("--drop", action="store",
                        default=DROP, type=float,
                        help="ratio of dropout used in fully connected layers")
    parser.add_argument("--loss", action="store",
                        default=LOSS,
                        help="keras loss function to use: mse, ...")
    parser.add_argument("--pool", action="store",
                        default=POOL, type=int,
                        help="pooling layer length")
    parser.add_argument("--scaling", action="store",
                        default=SCALING,
                        help="type of feature scaling; 'minabs': to [-1,1]; 'minmax': to [0,1], 'std': standard unit normalization; None: no normalization")
    parser.add_argument("--drug_features", action="store",
                        default="descriptors",
                        help="use dragon7 descriptors, latent representations from Aspuru-Guzik's SMILES autoencoder, or both, or random features; 'descriptors','latent', 'both', 'noise'")
    parser.add_argument("--feature_subsample", action="store",
                        default=FEATURE_SUBSAMPLE, type=int,
                        help="number of features to randomly sample from each category (cellline expression, drug descriptors, etc), 0 means using all features")
    parser.add_argument("--min_logconc", action="store",
                        default=MIN_LOGCONC, type=float,
                        help="min log concentration of dose response data to use: -3.0 to -7.0")
    parser.add_argument("--max_logconc", action="store",
                        default=MAX_LOGCONC, type=float,
                        help="max log concentration of dose response data to use: -3.0 to -7.0")
    parser.add_argument("--subsample", action="store",
                        default='naive_balancing',
                        help="dose response subsample strategy; None or 'naive_balancing'")
    parser.add_argument("--category_cutoffs", action="store", nargs='+', type=float,
                        default=CATEGORY_CUTOFFS,
                        help="list of growth cutoffs (between -1 and +1) seperating non-response and response categories")
    parser.add_argument("--train_samples", action="store",
                        default=0, type=int,
                        help="overrides the number of training samples if set to nonzero")
    parser.add_argument("--val_samples", action="store",
                        default=0, type=int,
                        help="overrides the number of validation samples if set to nonzero")
    parser.add_argument("--save", action="store",
                        default='save',
                        help="prefix of output files")
    parser.add_argument("--scramble", action="store_true",
                        help="randomly shuffle dose response data")
    parser.add_argument("--workers", action="store",
                        default=NB_WORKER, type=int,
                        help="number of data generator workers")

    return parser

def extension_from_parameters(args):
    """Construct string for saving model with annotation of parameters"""
    ext = ''
    ext += '.A={}'.format(args.activation)
    ext += '.B={}'.format(args.batch_size)
    ext += '.D={}'.format(args.drop)
    ext += '.E={}'.format(args.epochs)
    if args.feature_subsample:
        ext += '.F={}'.format(args.feature_subsample)
    if args.convolution:
        name = 'LC' if args.locally_connected else 'C'
        layer_list = list(range(0, len(args.convolution), 3))
        for l, i in enumerate(layer_list):
            nb_filter = args.convolution[i]
            filter_len = args.convolution[i+1]
            stride = args.convolution[i+2]
            if nb_filter <= 0 or filter_len <= 0 or stride <= 0:
                break
            ext += '.{}{}={},{},{}'.format(name, l+1, nb_filter, filter_len, stride)
        if args.pool and layer_list[0] and layer_list[1]:
            ext += '.P={}'.format(args.pool)
    for i, n in enumerate(args.dense):
        if n:
            ext += '.D{}={}'.format(i+1, n)
    ext += '.S={}'.format(args.scaling)

    return ext

def evaluate_keras_metric(y_true, y_pred, metric):
    objective_function = metrics.get(metric)
    objective = objective_function(y_true, y_pred)
    return K.eval(objective)

def evaluate_model(model, generator, samples, metric, category_cutoffs=[0.]):
    y_true, y_pred = None, None
    count = 0
    while count < samples:
        x_batch, y_batch = next(generator)
        y_batch_pred = model.predict_on_batch(x_batch)
        y_batch_pred = y_batch_pred.ravel()
        y_true = np.concatenate((y_true, y_batch)) if y_true is not None else y_batch
        y_pred = np.concatenate((y_pred, y_batch_pred)) if y_pred is not None else y_batch_pred
        count += len(y_batch)

    loss = evaluate_keras_metric(y_true.astype(np.float32), y_pred.astype(np.float32), metric)

    y_true_class = np.digitize(y_true, category_cutoffs)
    y_pred_class = np.digitize(y_pred, category_cutoffs)

    # theano does not like integer input
    acc = evaluate_keras_metric(y_true_class.astype(np.float32), y_pred_class.astype(np.float32), 'binary_accuracy')  # works for multiclass labels as well

    return loss, acc, y_true, y_pred, y_true_class, y_pred_class

def plot_error(y_true, y_pred, batch, file_ext, file_pre='save', subsample=1000):
    if batch % 10:
        return

    total = len(y_true)
    if subsample and subsample < total:
        usecols = np.random.choice(total, size=subsample, replace=False)
        y_true = y_true[usecols]
        y_pred = y_pred[usecols]

    y_true = y_true * 100
    y_pred = y_pred * 100
    diffs = y_pred - y_true

    bins = np.linspace(-200, 200, 100)
    if batch == 0:
        y_shuf = np.random.permutation(y_true)
        plt.hist(y_shuf - y_true, bins, alpha=0.5, label='Random')

    #plt.hist(diffs, bins, alpha=0.35-batch/100., label='Epoch {}'.format(batch+1))
    plt.hist(diffs, bins, alpha=0.3, label='Epoch {}'.format(batch+1))
    plt.title("Histogram of errors in percentage growth")
    plt.legend(loc='upper right')
    plt.savefig(file_pre+'.histogram'+file_ext+'.b'+str(batch)+'.png')
    plt.close()

    # Plot measured vs. predicted values
    fig, ax = plt.subplots()
    plt.grid('on')
    ax.scatter(y_true, y_pred, color='red', s=10)
    ax.plot([y_true.min(), y_true.max()],
            [y_true.min(), y_true.max()], 'k--', lw=4)
    ax.set_xlabel('Measured')
    ax.set_ylabel('Predicted')
    plt.savefig(file_pre+'.diff'+file_ext+'.b'+str(batch)+'.png')
    plt.close()

class MyLossHistory(Callback):
    def __init__(self, progbar, val_gen, test_gen, val_samples, test_samples, metric, category_cutoffs=[0.], ext='', pre='save'):
        super(MyLossHistory, self).__init__()
        self.progbar = progbar
        self.val_gen = val_gen
        self.test_gen = test_gen
        self.val_samples = val_samples
        self.test_samples = test_samples
        self.metric = metric
        self.category_cutoffs = category_cutoffs
        self.pre = pre
        self.ext = ext

    def on_train_begin(self, logs={}):
        self.best_val_loss = np.Inf
        self.best_val_acc = -np.Inf
        self.best_model = None

    def on_epoch_end(self, batch, logs={}):
        if float(logs.get('val_loss', 0)) < self.best_val_loss:
            self.best_model = self.model
            val_loss, val_acc, y_true, y_pred, y_true_class, y_pred_class = evaluate_model(self.best_model, self.val_gen, self.val_samples, self.metric, self.category_cutoffs)
            test_loss, test_acc, _, _, _, _ = evaluate_model(self.best_model, self.test_gen, self.test_samples, self.metric, self.category_cutoffs)
            self.progbar.append_extra_log_values([('val_acc', val_acc), ('test_loss', test_loss), ('test_acc', test_acc)])
            plot_error(y_true, y_pred, batch, self.ext, self.pre)
        self.best_val_loss = min(float(logs.get('val_loss', 0)), self.best_val_loss)
        self.best_val_acc = max(float(logs.get('val_acc', 0)), self.best_val_acc)

class MyProgbarLogger(ProgbarLogger):
    def on_train_begin(self, logs=None):
        super(MyProgbarLogger, self).on_train_begin(logs)
        self.verbose = 1
        self.extra_log_values = []

    def append_extra_log_values(self, tuples):
        for k, v in tuples:
            self.extra_log_values.append((k, v))

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        for k in self.params['metrics']:
            if k in logs:
                self.log_values.append((k, logs[k]))
        for k, v in self.extra_log_values:
            self.log_values.append((k, v))
        if self.verbose:
            self.progbar.update(self.seen, self.log_values, force=True)

def main():
    parser = get_parser()
    args = parser.parse_args()
    print('Args:', args)

    loggingLevel = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(level=loggingLevel, format='')

    ext = extension_from_parameters(args)

    loader = p1b3.DataLoader(feature_subsample=args.feature_subsample,
                             scaling=args.scaling,
                             drug_features=args.drug_features,
                             scramble=args.scramble,
                             min_logconc=args.min_logconc,
                             max_logconc=args.max_logconc,
                             subsample=args.subsample,
                             category_cutoffs=args.category_cutoffs)

    gen_shape = None
    out_dim = 1

    model = Sequential()
    if args.convolution and args.convolution[0]:
        gen_shape = 'add_1d'
        layer_list = list(range(0, len(args.convolution), 3))
        for l, i in enumerate(layer_list):
            nb_filter = args.convolution[i]
            filter_len = args.convolution[i+1]
            stride = args.convolution[i+2]
            if nb_filter <= 0 or filter_len <= 0 or stride <= 0:
                break
            if args.locally_connected:
                model.add(LocallyConnected1D(nb_filter, filter_len, subsample_length=stride, input_shape=(loader.input_dim, 1), activation=args.activation))
            else:
                model.add(Convolution1D(nb_filter, filter_len, subsample_length=stride, input_shape=(loader.input_dim, 1), activation=args.activation))
            if args.pool:
                model.add(MaxPooling1D(pool_length=args.pool))
        model.add(Flatten())

    for layer in args.dense:
        if layer:
            model.add(Dense(layer, input_dim=loader.input_dim, activation=args.activation))
            if args.drop:
                model.add(Dropout(args.drop))
    model.add(Dense(out_dim))

    model.summary()
    model.compile(loss=args.loss, optimizer=args.optimizer)

    train_gen = p1b3.DataGenerator(loader, batch_size=args.batch_size, shape=gen_shape).flow()
    val_gen = p1b3.DataGenerator(loader, partition='val', batch_size=args.batch_size, shape=gen_shape).flow()
    val_gen2 = p1b3.DataGenerator(loader, partition='val', batch_size=args.batch_size, shape=gen_shape).flow()
    test_gen = p1b3.DataGenerator(loader, partition='test', batch_size=args.batch_size, shape=gen_shape).flow()

    train_samples = int(loader.n_train/args.batch_size) * args.batch_size
    val_samples = int(loader.n_val/args.batch_size) * args.batch_size
    test_samples = int(loader.n_test/args.batch_size) * args.batch_size

    train_samples = args.train_samples if args.train_samples else train_samples
    val_samples = args.val_samples if args.val_samples else val_samples

    checkpointer = ModelCheckpoint(filepath=args.save+'.model'+ext+'.h5', save_best_only=True)
    progbar = MyProgbarLogger()
    history = MyLossHistory(progbar=progbar, val_gen=val_gen2, test_gen=test_gen,
                            val_samples=val_samples, test_samples=test_samples,
                            metric=args.loss, category_cutoffs=args.category_cutoffs,
                            ext=ext, pre=args.save)

    model.fit_generator(train_gen, train_samples,
                        nb_epoch=args.epochs,
                        validation_data=val_gen,
                        nb_val_samples=val_samples,
                        verbose=0,
                        callbacks=[checkpointer, history, progbar],
                        pickle_safe=True,
                        nb_worker=args.workers)

if __name__ == '__main__':
    main()
    try:
        K.clear_session()
    except AttributeError:      # theano does not have this function
        pass


'''
Created on Aug 17, 2016

@author: jvazquez
'''
import logging
from os.path import abspath, join, dirname

from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
from flask import Blueprint, g, jsonify, request, render_template
from flasgger import swag_from
from flask_jwt import jwt_required

from resource.api.allocation.allocation_period_response import \
    allocation_period_response
from resource.api.allocation.model import AllocationPeriod, Status, FREE
from resource.api.allocation.scheduler_layer import Schedule
from utils.helpers.response_creator.simple_creator import simplest
from utils.helpers.response_creator.http_statuses import CREATED, ERROR, OK
from resource.api.allocation.scheduler_layer import Schedule
from utils.helpers.acl.decorators import access_to_resource

template_folder = join(abspath(dirname(__file__)), "templates")
static_folder = join(abspath(dirname(__file__)), "static")
allocation_module = Blueprint(__name__, "allocation",
                              template_folder=template_folder,
                              static_url_path="/static/allocation",
                              static_folder=static_folder)
logger = logging.getLogger(__name__)
logger.debug("Template folder: {}".format(template_folder))

@allocation_module.route("/api/allocation/", methods=["POST"])
@jwt_required(401)
@access_to_resource(request, "allocation.create")
@swag_from("create.yml")
def create():
    scheduler = Schedule()
    data = request.get_json()
    errors = []
    alloc_list = []
    for single_data in data:
        start_time = parse(single_data["start_time"])
        end_time = parse(single_data["end_time"])
        resource_id = single_data["resource_id"]
        validation = scheduler.validator.are_dates_sane(start_time, end_time)
        if validation:
            single_data = scheduler.sanitize_data(single_data)
            alloc_id = scheduler.create(single_data, errors)
            if alloc_id:
                alloc_list.append(alloc_id)
        else:
            errors.append(single_data)

    if errors:
        # TODO: Modificar el error porque no corresponde!
        response = simplest("custom", 417, payload=None,
                            rejected_fields=errors,
                            callback=allocation_period_response)
    else:
        if not alloc_list:
            logger.error("There was an error when save the allocation for this resource: {}".format(resource_id))
            logger.error("This user did the allocation with error: {}".format(single_data["user_id"]))
        response = simplest("create", CREATED,
                            payload=alloc_list, rejected_fields=errors,
                            normal_mode=False)

    return jsonify(response), OK

@allocation_module.route("/api/allocation/<int:allocation_id>/",
                         methods=["PATCH"])
@jwt_required(401)
@access_to_resource(request, "allocation.update")
@swag_from("update.yml")
def update(allocation_id):
    errors = []
    scheduler = Schedule()
    data = request.get_json()
    data = scheduler.sanitize_data(data)
    validation = scheduler.validator.are_dates_sane(data["start_time"], data["end_time"])
    if validation:
        scheduler.update(allocation_id, data, errors)
    else:
        errors.append(data)

    if errors:
        # TODO: Modificar el error porque no corresponde!
        response = simplest("custom", 417, payload=None,
                            rejected_fields=errors,
                            callback=allocation_period_response)
    else:
        response = simplest("update", OK)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/<int:allocation_id>/",
                         methods=["GET"])
@jwt_required(401)
@access_to_resource(request, "allocation.get")
@swag_from("get.yml")
def get(allocation_id):
    scheduler = Schedule()
    allocation = scheduler.get_alloc_by_id(allocation_id)
    response = simplest("get", OK, allocation)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/all/<int:user_id>", methods=["GET"])
@jwt_required(401)
@access_to_resource(request, "allocation.get_all")
@swag_from("get_all.yml")
def get_all(user_id):
    scheduler = Schedule()
    allocation_list = scheduler.get_alloc_by_user_id(user_id)
    response = simplest("get_all", OK, allocation_list)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/all_resource/<string:resource_id>", methods=["GET"])
@jwt_required(401)
@access_to_resource(request, "allocation.get_all")
@swag_from("get_all_resource.yml")
def get_all_by_resource(resource_id):
    scheduler = Schedule()
    allocation_list = scheduler.get_alloc_by_resource_id(resource_id)
    response = simplest("get_all", OK, allocation_list)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/<int:period_id>/",
                         methods=["DELETE"])
@jwt_required(401)
@access_to_resource(request, "allocation.delete")
@swag_from("delete.yml")
def delete(period_id):
    scheduler = Schedule()
    errors = []
    scheduler.delete_period(period_id, errors)
    if errors:
        # TODO: Modificar el error porque no corresponde!
        response = simplest("custom", 417, payload=None,
                            rejected_fields=errors,
                            callback=allocation_period_response)
    else:
        response = simplest("delete", OK, None)
    return jsonify(response), OK


import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torch
import torchaudio
import torchaudio.transforms as transforms
import torchvision.transforms as tvt
import torch.optim as optim

import models
from tqdm import tqdm
import os
import random
import math
import librosa

import argparse
import mat_transforms

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--encoder', type=str, default=None,
                    help='Encoder path')
parser.add_argument('--decoder', type=str, default=None,
                    help='Decoder path')
parser.add_argument('--style', type=str, default=None,
                    help='Style image path')
parser.add_argument('--content', type=str, default=None,
                    help='Content image path')
parser.add_argument('--x', type=int, default=1,
                    help='Number of AE layers to use')

args = parser.parse_args()

def load_audio(fname, transform):
    # output, _ = torchaudio.load(fname, normalization=True)

    # output = transforms.MelSpectrogram(sample_rate=16000,
    #                                    n_fft=400, win_length=400,
    #                                    hop_length=160, n_mels=128)(output)
    y, _ = librosa.load(fname, 16000)
    stft_y = librosa.core.stft(y, n_fft=400, hop_length=160, win_length=400)
    mag = np.abs(stft_y)
    ang = np.angle(stft_y)
    D = mag**2
    output = librosa.feature.melspectrogram(S=D, sr=16000)

    output = torch.Tensor(output)
    output = output.unsqueeze(0)

    output = output[:, :, :1000]
    ang = ang[:, :1000]
    if transform is not None:
        output = transform(output)

    # output = output.unsqueeze(0)  # [1, 128, 1000]
    return output, ang

transform = tvt.Compose(
        [transforms.AmplitudeToDB(stype='power', top_db=None),
         tvt.Normalize(mean=[-38.39992], std=[13.462255])])

reverse_normalize = tvt.Normalize(
    mean=[38.39992/13.462255],
    std=[1./13.462255])

num_layers = args.x

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Assuming that we are on a CUDA machine, this should print a CUDA device:

print(device)

encoder = models.encoder(x=num_layers, pretrained_path=args.encoder).to(device)
decoder = models.decoder(x=num_layers, pretrained_path=args.decoder).to(device)

encoder.train(False)
decoder.train(False)

content_audio, content_ang = load_audio(args.content, transform)
content_audio = content_audio.to(device)
style_audio, _ = load_audio(args.style, transform)
style_audio = style_audio.to(device)

z_content, maxpool_content = encoder(content_audio)  # (1, C, W)
z_style, _ = encoder(style_audio)  # (1, C, W)

n_channels = z_content.size()[1]  # C
n_1 = z_content.size()[2]  # W

z_content = z_content.squeeze(0) # .view([n_channels, -1])  # (C, HW)
z_style = z_style.squeeze(0) # .view([n_channels, -1])  # (C, HW)

white_content = mat_transforms.whitening(z_content.cpu().detach().numpy())  # (C, HW)
color_content = mat_transforms.colouring(z_style.cpu().detach().numpy(), white_content)  # (C, HW)

# alpha = 0.6
# color_content = alpha*color_content + (1.-alpha)*z_content.cpu().detach().numpy()

color_content = torch.Tensor(color_content) # tvt.ToTensor()(color_content)
# color_content = color_content.view([n_channels, n_1, n_2]) # (C, H, W)
color_content = color_content.unsqueeze(0) # (1, C, W)

inputs_hat = decoder(color_content.to(device), maxpool_content)

new_audio = inputs_hat # .squeeze(0) # (1, C, W)
new_audio = reverse_normalize(new_audio) # (1, C, W)
new_audio = new_audio[0] # take only 1 channel (C, W)

new_audio = np.maximum(np.minimum(new_audio.cpu().detach().numpy(), 0.0), -80.0)
power_spectro = librosa.core.db_to_power(new_audio, ref=1.0)

inv_power = librosa.feature.inverse.mel_to_stft(power_spectro, sr=16000, n_fft=400, power=2.0)

full_stft = inv_power*np.exp(1j*content_ang)
y_new = librosa.core.istft(full_stft, hop_length=160, win_length=400)
# y_new = librosa.feature.inverse.mel_to_audio(power_spectro, sr=16000,
#                                              n_fft=400, hop_length=160,
#                                              win_length=400, n_iter=32) # it will take sqrt

librosa.output.write_wav("processed.wav", y_new, 16000)


# Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import Callable, Dict, Tuple

import torch
from torch import nn

from flash.core.data.io.input import DataKeys
from flash.core.data.transforms import ApplyToKeys, kornia_collate, merge_transforms
from flash.core.utilities.imports import _ALBUMENTATIONS_AVAILABLE, _KORNIA_AVAILABLE, _TORCHVISION_AVAILABLE, requires

if _KORNIA_AVAILABLE:
    import kornia as K

if _TORCHVISION_AVAILABLE:
    import torchvision
    from torchvision import transforms as T

if _ALBUMENTATIONS_AVAILABLE:
    import albumentations

class AlbumentationsAdapter(torch.nn.Module):
    @requires("albumentations")
    def __init__(self, transform):
        super().__init__()
        if not isinstance(transform, list):
            transform = [transform]
        self.transform = albumentations.Compose(transform)

    def forward(self, x):
        return torch.from_numpy(self.transform(image=x.numpy())["image"])

def default_transforms(image_size: Tuple[int, int]) -> Dict[str, Callable]:
    """The default transforms for image classification: resize the image, convert the image and target to a tensor,
    collate the batch, and apply normalization."""
    if _KORNIA_AVAILABLE and os.getenv("FLASH_TESTING", "0") != "1":
        #  Better approach as all transforms are applied on tensor directly
        return {
            "to_tensor_transform": nn.Sequential(
                ApplyToKeys(DataKeys.INPUT, torchvision.transforms.ToTensor()),
                ApplyToKeys(DataKeys.TARGET, torch.as_tensor),
            ),
            "post_tensor_transform": ApplyToKeys(
                DataKeys.INPUT,
                K.geometry.Resize(image_size),
            ),
            "collate": kornia_collate,
            "per_batch_transform_on_device": ApplyToKeys(
                DataKeys.INPUT,
                K.augmentation.Normalize(torch.tensor([0.485, 0.456, 0.406]), torch.tensor([0.229, 0.224, 0.225])),
            ),
        }
    return {
        "pre_tensor_transform": ApplyToKeys(DataKeys.INPUT, T.Resize(image_size)),
        "to_tensor_transform": nn.Sequential(
            ApplyToKeys(DataKeys.INPUT, torchvision.transforms.ToTensor()),
            ApplyToKeys(DataKeys.TARGET, torch.as_tensor),
        ),
        "post_tensor_transform": ApplyToKeys(
            DataKeys.INPUT,
            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ),
        "collate": kornia_collate,
    }

def train_default_transforms(image_size: Tuple[int, int]) -> Dict[str, Callable]:
    """During training, we apply the default transforms with additional ``RandomHorizontalFlip``."""
    if _KORNIA_AVAILABLE and os.getenv("FLASH_TESTING", "0") != "1":
        #  Better approach as all transforms are applied on tensor directly
        transforms = {
            "post_tensor_transform": ApplyToKeys(DataKeys.INPUT, K.augmentation.RandomHorizontalFlip()),
        }
    else:
        transforms = {"pre_tensor_transform": ApplyToKeys(DataKeys.INPUT, T.RandomHorizontalFlip())}
    return merge_transforms(default_transforms(image_size), transforms)


"""
Author: Kyle Guthrie
Date: 05/12/15
Description: Implements the mechanics of Bubble Sort, but limited to one pass. Counts the number of swap operations.
"""

def code_abbey_array_checksum(valueList):
	# Checksum algorithm from Problem 17
	checksum = 0
	for item in valueList:
		checksum = ((checksum + item)*113) % 10000007
	return checksum

def main():
	with open("input.txt","r") as fidInput:
		valueList = [int(item) for item in fidInput.readline().split()]
		# remove trailing -1 that marks end of array (not necessary in this input format)
		valueList = valueList[:-1]
		swaps = 0
		for idx in range(0,len(valueList)-1):
			if valueList[idx] > valueList[idx+1]:
				valueList[idx],valueList[idx+1] = valueList[idx+1],valueList[idx]
				#print( *valueList )
				swaps += 1
		# compute checksum:
		checksum = code_abbey_array_checksum(valueList)

	with open("output.txt","w") as fidOutput:
		print(swaps,checksum, file=fidOutput)

if __name__ == '__main__':
	main()


#Queue implementation using List
stockPrice=[]
stockPrice.insert(0,100)
stockPrice.insert(0,131)
stockPrice.insert(0,155)
#Since i am inserting in 0th position everytime, the previous datas are shifted to right
print(stockPrice)
print(stockPrice.pop())

#Instead of list as queue, we can use deque module

from collections import deque
q=deque()
#deque can be used as both queue and stack. appendleft for queu whereas append for stack
q.appendleft(100)
q.appendleft(20)
q.appendleft(250)
print(q.pop())


from board.widgets import containers
from board.widgets import video
from board.widgets import typography
from board.core.plugin import Plugin
import inspect

class DemoEmbeddedVideo(Plugin):
    title = "Embedded video"
    category = "Extra"

    def __init__(self):
        super().__init__()

    async def init_page(self, **kw):
        full_container = containers.Container()
        hstack = containers.Container(orient="horizontal", max_per_row=2)
        container = containers.Container()
        container.append(typography.Title(
            "Thinking about Concurrency, Raymond Hettinger",
            level=2
        ))
        container.append(video.Youtube("Bv25Dwe84g0"))

        container.append(typography.Paragraph(
            "To go further: "
            "https://developers.google.com/youtube/iframe_api_reference"
        ))

        hstack << container

        container = containers.Container()
        container.append(typography.Title(
            "Rempaillage cannage prés des yvelines versailles houdan",
            level=2
        ))
        container.append(video.Dailymotion("xgshqb"))
        container.append(typography.Paragraph(
            "To go further: "
            "http://dailymotion.canalblog.com/archives/2011/02/10/20351553.html"
        ))

        hstack << container
        full_container << hstack
        full_container << typography.Title("The source", level=2)
        full_container << code.MySource()

        return full_container


class Solution:
    def threeSumClosest(self, nums, target):
        """
        :type nums: List[int]
        :type target: int
        :rtype: int
        """
        if nums is None or len(nums) == 0 or target is None:
            return []

        nums.sort()
        minsum = None
        import sys
        mindiff = sys.maxsize
        for left in range(len(nums) - 2):
            l, r = left + 1, len(nums) - 1
            while l < r:
                csum = nums[left] + nums[l] + nums[r]
                if csum == target:
                    return csum
                if csum < target:
                    l += 1
                else:
                    r -= 1
                if abs(csum - target) < mindiff:
                    mindiff = abs(csum - target)
                    minsum = csum

        return minsum

if __name__ == "__main__":
    sln = Solution()
    #res = sln.threeSumClosest([-1,2,1,-4], 1)
    #res = sln.threeSumClosest([1,1,1,0], -100)
    res = sln.threeSumClosest([0,2,1,-3], 1)
    print(res)


import inline as inline
import matplotlib
import numpy as np

#%matplotlib inline

from matplotlib import pyplot as plt

# набор данных ( необходимо для работы sgd)
X = np.array([
    [-2, 4, -1],
    [4, 1, -1],
    [1, 6, -1],
    [2, 4, -1],
    [6, 2, -1],

])

y = np.array([-1, -1, 1, 1, 1])

# стохастический градиентынй спуск
def perceptron_sgd(X, Y):
    w = np.zeros(len(X[0]))     # инициализация вектора весов
    eta = 1                     # скорость обучения
    epochs = 20                 # количсество эпох

    for t in range(epochs):                    # итерация n раз
        for i, x in enumerate(X):              # итерация по каждой выборке
            if (np.dot(X[i], w) * Y[i]) <= 0:  # условие неправильности классификации
                w = w + eta * X[i] * Y[i]      # обновление правила для весов

    return w

w = perceptron_sgd(X, y)
print(w)

for d, sample in enumerate(X):

    if d < 2:
        plt.scatter(sample[0], sample[1], s=120, marker='_', linewidths=2)

    else:
        plt.scatter(sample[0], sample[1], s=120, marker='+', linewidths=2)

plt.plot([-2, 6], [6, 0.5])     # Построить график
plt.show()                      # Отобразить график

# Учится (2 график)
def perceptron_sgd_plot(X, Y):
    '''
    train perceptron and plot the total loss in each epoch.

    :param X: data samples
    :param Y: data labels
    :return: weight vector as a numpy array
    '''
    w = np.zeros(len(X[0]))
    eta = 1
    n = 30
    errors = []

    for t in range(n):
        total_error = 0
        for i, x in enumerate(X):
            if (np.dot(X[i], w) * Y[i]) <= 0:
                total_error += (np.dot(X[i], w) * Y[i])
                w = w + eta * X[i] * Y[i]
        errors.append(total_error * -1)  # подсчет ошибок в каждой эпохе

    plt.plot(errors)
    plt.xlabel('Epoch')
    plt.ylabel('Total Loss')
    plt.show()

    return w

print(perceptron_sgd_plot(X, y))

# Evaluation
for d, sample in enumerate(X):
    # Plot the negative samples
    if d < 2:
        plt.scatter(sample[0], sample[1], s=120, marker='_', linewidths=2)
    # Plot the positive samples
    else:
        plt.scatter(sample[0], sample[1], s=120, marker='+', linewidths=2)

# добавить тестовые значения
plt.scatter(2, 2, s=120, marker='_', linewidths=2, color='yellow')
plt.scatter(4, 3, s=120, marker='+', linewidths=2, color='blue')

# Print the hyperplane calculated by perceptron_sgd()
x2 = [w[0], w[1], -w[1], w[0]]
x3 = [w[0], w[1], w[1], -w[0]]

x2x3 = np.array([x2, x3])
X, Y, U, V = zip(*x2x3)
ax = plt.gca()
ax.quiver(X, Y, U, V, scale=1, color='blue')

plt.show()


import http
import os

from resource_api.common.constants import Constants
from resource_api.common.dynamo import DynamoDB
from resource_api.common.helpers import response

from resource_api.modify_resource.main.RequestHandler import RequestHandler

_dynamodb = None

def handler(event, context):
    """
    Handler method for modify resource function.
    """
    if event is None:
        return response(http.HTTPStatus.BAD_REQUEST, Constants.error_insufficient_parameters())
    if event is None or Constants.event_body() not in event or Constants.event_http_method() not in event:
        return response(http.HTTPStatus.BAD_REQUEST, Constants.error_insufficient_parameters())
    if event[Constants.event_body()] is None or len(event[Constants.event_body()]) is 0:
        return response(http.HTTPStatus.BAD_REQUEST, Constants.error_insufficient_parameters())

    global _dynamodb
    if _dynamodb is None:
        try:
            ddb = DynamoDB()
            _dynamodb = ddb.connect(os.environ[Constants.env_var_region()])
        except Exception as e:
            return response(http.HTTPStatus.INTERNAL_SERVER_ERROR, str(e))

    try:
        request_handler = RequestHandler(_dynamodb)
    except Exception as e:
        return response(http.HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
    return request_handler.handler(event, context)

def clear_dynamodb():
    """
    Clear the global dynamodb instance.
    """
    globals()['_dynamodb'] = None


# -*- coding: utf-8 -*-

import jieba
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.cluster import KMeans
import pandas as pd
import random
import numpy as np

class KmeansClustering():
    def __init__(self, stopwords_path=None):
        self.stopwords = self.load_stopwords(stopwords_path)
        self.vectorizer = CountVectorizer()
        self.transformer = TfidfTransformer()

    def load_stopwords(self, stopwords=None):
        """
        加载停用词
        :param stopwords:
        :return:
        """
        if stopwords:
            with open(stopwords, 'r', encoding='utf-8') as f:
                return [line.strip() for line in f]
        else:
            return []

    def preprocess_data(self, contents):
        """
        文本预处理，每行一个文本
        :param corpus_path:
        :return:
        """
        corpus = []

        for line in contents:
            corpus.append(' '.join([word for word in jieba.lcut(line.strip()) if word not in self.stopwords]))
        return corpus

    def get_text_tfidf_matrix(self, corpus):
        """
        获取tfidf矩阵
        :param corpus:
        :return:
        """
        tfidf = self.transformer.fit_transform(self.vectorizer.fit_transform(corpus))

        # 获取词袋中所有词语
        # words = self.vectorizer.get_feature_names()

        # 获取tfidf矩阵中权重
        from scipy.sparse import coo_matrix
        weights = coo_matrix(tfidf, dtype=np.float64).toarray()

        return weights

    def kmeans(self, corpus_path, n_clusters=5):
        """
        KMeans文本聚类
        :param corpus_path: 语料路径（每行一篇）,文章id从0开始
        :param n_clusters: ：聚类类别数目
        :return: {cluster_id1:[text_id1, text_id2]}
        """
        corpus = self.preprocess_data(corpus_path)
        weights = self.get_text_tfidf_matrix(corpus)

        clf = KMeans(n_clusters=n_clusters)

        # clf.fit(weights)

        y = clf.fit_predict(weights)

        # 中心点
        # centers = clf.cluster_centers_

        # 用来评估簇的个数是否合适,距离约小说明簇分得越好,选取临界点的簇的个数
        # score = clf.inertia_

        # 每个样本所属的簇

        semi["class_label"] = y

        result = {}
        for text_idx, label_idx in enumerate(y):
            if label_idx not in result:
                result[label_idx] = [text_idx]
            else:
                result[label_idx].append(text_idx)
        return result

if __name__ == '__main__':

    random.seed(42)
    np.random.seed(42)

    Kmeans = KmeansClustering(stopwords_path='../data/stop_words.txt')
    semi = pd.read_csv("../data/unlabeled_data.csv")
    result = Kmeans.kmeans(semi["content"], n_clusters=10)
    semi.to_csv("semi.csv", index=False)
    print(result)


#!/usr/bin/python

import numpy
import pylab

pylab.rcParams["font.size"]=8
pylab.rcParams["legend.fontsize"]=6
#pylab.rcParams["lines.linewidth"]=1
#pylab.rcParams["axes.linewidth"]=2
#pylab.rcParams["axes.labelsize"]="large"
#pylab.rcParams["axes.labelweight"]="bold"
pylab.rcParams["xtick.major.size"]=0
pylab.rcParams["xtick.minor.size"]=0
pylab.rcParams["ytick.major.size"]=0
pylab.rcParams["ytick.minor.size"]=0
#pylab.rcParams["xtick.direction"]="out"
#pylab.rcParams["ytick.direction"]="out"
pylab.rcParams["figure.figsize"]=(2, 2.5)

#inputs
som_input=numpy.loadtxt("som_input.csv", delimiter=",")
dnd_input=numpy.loadtxt("dnd_input.csv", delimiter=",")

plot_end=500

pylab.clf()
pylab.subplot(2,1,1)
for i in range(1,som_input.shape[1]):
    pylab.plot(som_input[:plot_end,0], som_input[:plot_end,i])
pylab.ylim([0.0, 0.05])
pylab.yticks([0.0, 0.05])
pylab.xticks([round(som_input[0,0]), round(som_input[plot_end,0])])
pylab.ylabel("Input (soma)")
pylab.xlabel("Time [s]")

pylab.subplot(2,1,2)
for i in range(1,dnd_input.shape[1]):
    pylab.plot(dnd_input[:plot_end,0], dnd_input[:plot_end,i])
pylab.ylim([0.0, 0.05])
pylab.yticks([0.0, 0.05])
pylab.xticks([round(dnd_input[0,0]), round(dnd_input[plot_end,0])])
pylab.ylabel("Input (dendrite)")
pylab.xlabel("Time [s]")

#pylab.xticks([])
#pylab.ylim([0.0, 1.1])
#pylab.yticks([0.0, 1.0])

pylab.tight_layout()
pylab.savefig("input.pdf")


# _*_coding:utf-8_*_
# @Time    : 2020/10/28 0:55

# https://leetcode-cn.com/problems/longest-substring-without-repeating-characters/

# res初始化为0，建立两个索引i和j，判断新来的字符，是否在s[i:j]中出现
# 若没有出现，更新res，取之前res与i到j的最大值
# 若出现了，则更新i索引
class Solution:
    def lengthOfLongestSubstring(self, s: str) -> int:
        res = i = 0
        for j in range(len(s)):     #建立两个索引i和j，判断新来的字符，是否在s[i:j]中出现
            if s[j] not in s[i:j]:  # 若没有出现
                res = max(res, j+1-i)   # 更新res，取之前res与i到j的最大值
            else:   # 若出现了
                i = i + s[i:j].index(s[j]) + 1  #更新i的位置
        return res

0003 无重复字符最长子串.py

0003 无重复字符最长子串.py


import os
from setuptools import setup

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

classifiers = """\
Development Status :: alpha
Environment :: Console
Intended Audience :: Science/Research
Intended Audience :: Developers
License :: GPL
Operating System :: OS Independent
Programming Language :: Python
Topic :: Scientific/Engineering
Topic :: Software Development :: Libraries :: Python Modules
"""

setup(
    name = "PyFVCOM",
    version = "1.0",
    author = "Pierre Cazenave",
    author_email = "pica@pml.ac.uk",
    description = ("PyFVCOM is a collection of various tools and utilities which can be used to extract, analyse and plot input and output files from FVCOM."),
    license = "MIT",
    keywords = "fvcom",
    platforms = "any",
    url = "https://gitlab.ecosystem-modelling.pml.ac.uk/pica/PyFVCOM",
    packages=['PyFVCOM'],
    long_description=read('README.md'),
    classifiers=classifiers
)