import numpy as np
import torch
import pickle
from HEnet import HENet
from sampler import Sampler
from torch import nn, optim
from torch.autograd import Variable
from sampler import Sampler
from pathlib import Path
from dataset import FallData, Flip, Scale, Translate
from dataset import Normalize, OutsideIamge, RemoveJoints
from torch.utils.data import DataLoader
from torchvision.transforms import transforms
import warnings
warnings.filterwarnings("ignore")
def main():
  Path("saved_model").mkdir(exist_ok=True)
  device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
  batch_size = 64
  num_epochs = 100
  test_step = 50
  with open('TrainAndTestData/inputs.pkl','rb') as source: inputs = pickle.load(source)
  with open('TrainAndTestData/label.pkl','rb') as source: labels = pickle.load(source)
  with open('TrainAndTestData/inputs_test.pkl','rb') as source: test_inputs = pickle.load(source)
  with open('TrainAndTestData/label_test.pkl','rb') as source: test_labels = pickle.load(source)
  inputs = np.reshape(inputs, (inputs.shape[0], 32, 15, 3))
  inputs = np.transpose(inputs, (0, 3, 1, 2))
  inputs[:, 2, :, :] = 0
  labels = np.asarray(labels,dtype=np.int32)
  test_inputs = np.reshape(test_inputs, (test_inputs.shape[0], 32, 15, 3))
  test_inputs = np.transpose(test_inputs, (0, 3, 1, 2))
  test_inputs[:, 2, :, :] = 0
  test_labels = np.asarray(test_labels,dtype=np.int32)
  train_dataset = FallData(inputs,labels,
                     transform=transforms.Compose([
                         OutsideIamge(),
                         RemoveJoints(),
                         Scale(),
                         Translate(),
                         Flip(),
                         Normalize()
                     ]))
  test_dataset = FallData(test_inputs,test_labels,
                       transform=transforms.Compose([
                           RemoveJoints(),
                           Normalize()
                       ]))

  datasampler = Sampler(train_dataset)
  train_loader = DataLoader(train_dataset, batch_size=batch_size,sampler = datasampler)
  test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)
  Encrypted_model = HENet(10)
  Encrypted_model.to(device)
  CrossEntropyLoss = nn.CrossEntropyLoss()
  optimizer = optim.SGD(Encrypted_model.parameters(), lr = 0.01, momentum = 0.9, weight_decay = 5e-4)
  scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
  best_cost = 100
  for epoch in range(num_epochs):
    costs=0
    Encrypted_model.train()
    scheduler.step()
    for i, data in enumerate(train_loader):
      train_data = Variable(data["x"]).to(device)
      train_label = Variable(torch.tensor(data["y"], dtype=torch.long)).to(device)
      optimizer.zero_grad()
      outputs = Encrypted_model(train_data)
      cost = CrossEntropyLoss(outputs, train_label)
      cost.backward()
      costs += cost.item()
      optimizer.step()
    total_cost = costs / len(train_loader)
    if total_cost<best_cost:
      print ('\033[91m'+"Saving the model"+'\033[0m')
      torch.save(Encrypted_model.state_dict(), "saved_model/Fall_model.torch")
      best_cost=total_cost
    print("Epoch: %s, Loss: %s" %(epoch,total_cost))
    acc = 0
    if ((epoch+1) % test_step == 0):
      Encrypted_model.eval()
      for i, data in enumerate(test_loader):
        test_data = Variable(data["x"]).to(device)
        test_label = data["y"].numpy()[0]
        outputs = Encrypted_model(test_data).data.cpu().numpy()[0]
        predicts = np.argmax(outputs)
        acc += (predicts == test_label).sum()
      accuracy = acc / len(test_loader)
      print('\33[34m'+"Test set accuracy: %.2f" %(accuracy)+'\033[0m')
  print ('\033[91m'+"Saving the final model"+'\033[0m')
  torch.save(Encrypted_model.state_dict(), "saved_model/Fall_model_final.torch")
if __name__ == '__main__':
  main()


# 每日早上 01:00 更新前一日的全部数据
import pandas as pd
import os, time
from datetime import timedelta

# 获取日期
def get_time():
    t = time.localtime(time.time())
    d, m = t.tm_mday,  t.tm_mon
    return m, d
m, d = get_time()

# 判断是否已经更新
update_time = open('dxylastest/update','r').read()
if update_time != '2020-%02d-%02d' % (m, d):
    update_time = '2020-%02d-%02d' % (m, d)
    # 加载 lastest historical data
    df = pd.read_csv('dxylastest/city.csv')
    # 增量更新
    shell = "wget http://69.171.70.18:5000/data/city_level_2020-%02d-%02dT00.csv -P dxylastest/" % (m,d)
    os.system(shell)
    columns = ['date','provinceShortName','city.cityName','city.confirmedCount','city.curedCount','city.deadCount','city.suspectedCount',
               'confirmedCount','suspectedCount','curedCount','deadCount','comment']
    tmp = pd.read_json('dxylastest/' + 'city_level_2020-%02d-%02dT00.csv' % (m,d))
    tmp['date'] = update_time
    tmp['date'] = pd.to_datetime(tmp['date']) - timedelta(days=1)
    df = df.append(tmp[columns])
    #df.sort_values("date",inplace=True)
    df.drop_duplicates(inplace=True)
    df.to_csv('dxylastest/city.csv',index=0)
    # 导出 rds
    os.system('Rscript csv2rds.R')
    # clear
    open('dxylastest/update','w').write(update_time)
else:
    print('everything is updated')


import pytest
import sympy
import BondGraphTools as bgt
import BondGraphTools.sim_tools as sim

@pytest.mark.use_fixture("rlc")
def test_build(rlc):
    assert len(rlc.state_vars) == 2
    assert len(rlc.ports) == 0

@pytest.mark.use_fixture("rlc")
def test_build_and_drive(rlc):
    se = bgt.new("Se")
    assert len(se.control_vars) == 1
    rlc += se

    for comp in rlc.components.values():
        if comp.type == "0":
            rlc.connect(se, comp)
            break

    assert len(rlc.bonds) == 4
    assert len(rlc.control_vars) == 1

def test_symbolic_params():
    r = bgt.new("R", value=sympy.symbols('r'))
    l = bgt.new("I", value=sympy.symbols('l'))
    c = bgt.new("C", value=sympy.symbols('c'))
    kvl = bgt.new("0", name="kvl")

    rlc = r + l + c + kvl

    rlc.connect(r, kvl)
    rlc.connect(l, kvl)
    rlc.connect(c, kvl)

    assert len(rlc.params) == 3

    assert set(rlc.params.values()) & set(sympy.symbols('r, l, c'))

@pytest.mark.use_fixture("rlc")
def test_rlc_con_rel(rlc):

    rel = rlc.constitutive_relations

    eq1 = sympy.sympify("dx_0 - x_1")
    eq2 = sympy.sympify("dx_1 + x_0 + x_1")

    for r in rel:
        assert r in (eq1, eq2)

    assert "x_0" in rlc.state_vars
    assert "x_1" in rlc.state_vars

@pytest.mark.skip
@pytest.mark.usefixture("rlc")
def test_add_forcing(rlc):
    port = rlc.make_port()

    assert port == 0

    rlc.connect((rlc, port), "0_0")

    assert rlc.ports == {
        0: (rlc, port)
    }

    ts, ps, cv = rlc.basis_vectors

    assert len(ps) == 1

    assert rlc.constitutive_relations == list(
        sympy.sympify("dx_0 - x_1, dx_1 + f_0 + x_0 + x_1, e_0 - x_1")
    )

def test_tf():
    l = bgt.new("I", value=1)
    c = bgt.new("C", value=1)
    tf = bgt.new("TF", value=0.5)

    tflc = tf + l + c
    tflc.connect(l, (tf, 1))
    tflc.connect(c, (tf, 0))

    c,m,lp,nlp,const = tflc.system_model()
    assert nlp.is_zero
    assert const ==[]

def test_se():

    Se = bgt.new('Se', value=1)
    c = bgt.new('C', value=1)
    vc = Se + c

    assert Se.constitutive_relations == [sympy.sympify("e_0 - 1")]
    vc.connect(Se, c)

    assert vc.constitutive_relations == [sympy.sympify("dx_0"),
                                         sympy.sympify("x_0 - 1")]


# coding=utf-8
# --------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
#
# Code generated by Microsoft (R) AutoRest Code Generator.
# Changes may cause incorrect behavior and will be lost if the code is
# regenerated.
# --------------------------------------------------------------------------

from msrest.serialization import Model

class PeriodicTimerSourceInfo(Model):
    """Periodic timer event source.

    All required parameters must be populated in order to send to Azure.

    :param start_time: Required. The time of the day that results in a valid
     trigger. Schedule is computed with reference to the time specified up to
     seconds. If timezone is not specified the time will considered to be in
     device timezone. The value will always be returned as UTC time.
    :type start_time: datetime
    :param schedule: Required. Periodic frequency at which timer event needs
     to be raised. Supports daily, hourly, minutes, and seconds.
    :type schedule: str
    :param topic: Topic where periodic events are published to IoT device.
    :type topic: str
    """

    _validation = {
        'start_time': {'required': True},
        'schedule': {'required': True},
    }

    _attribute_map = {
        'start_time': {'key': 'startTime', 'type': 'iso-8601'},
        'schedule': {'key': 'schedule', 'type': 'str'},
        'topic': {'key': 'topic', 'type': 'str'},
    }

    def __init__(self, *, start_time, schedule: str, topic: str=None, **kwargs) -> None:
        super(PeriodicTimerSourceInfo, self).__init__(**kwargs)
        self.start_time = start_time
        self.schedule = schedule
        self.topic = topic


# Third parties imports
import pytest

# Local imports
from oblivion import functions

@pytest.mark.parametrize(
    "args,result", [
        ((b'test', 1), b'\x1b'),
        ((b'test', 2), b'\x1b\xc2'),
    ])
def test_mask_generation_function(args, result):
    """Test functions.mask_generation_function."""
    assert functions.mask_generation_function(*args) == result

@pytest.mark.parametrize(
    "args,result", [
        ((2,), (2, 3, 5, 7)),
        ((3,), (5, 7, 11, 13)),
    ])
def test_generate_prime(args, result):
    """Test functions.generate_prime."""
    assert functions.generate_prime(*args) in result


import numpy as np
import csv as csv
import theano, re, cPickle
import theano.tensor as T
import matplotlib.pyplot as plt

# How to run the script using GPU and proper floating point configuration
# THEANO_FLAGS='cuda.root=/usr/local/cuda-5.0/bin,device=gpu,floatX=float32' python [filename].py

theano.config.floatX = 'float32'

# Load the train dataset
f = file('../../data_lab/train_data.dat', 'rb')
train_data = np.array(cPickle.load(f), dtype=np.float32)
f.close()

# Separate the test dataset into train, cross, and test set.
train_set = train_data[0:534, 0::]
cross_set = train_data[534:712, 0::]
ttest_set = train_data[712::, 0::]

# Separate the features and the labels
# ==============================================================================
train_set_x = train_set[0::,1::]
train_set_y = train_set[0::,0]
# ==============================================================================
cross_set_x = cross_set[0::,1::]
cross_set_y = cross_set[0::,0]
# ==============================================================================
ttest_set_x = ttest_set[0::,1::]
ttest_set_y = ttest_set[0::,0]
# ==============================================================================

# Get the dimensions of the training dataset
num_examples = train_set_x.shape[0]
num_features = train_set_x.shape[1]

# Load the test dataset
f = file('../../data_lab/test_data.dat', 'rb')
test_set = np.array(cPickle.load(f), dtype=np.float32)
f.close()

# ==============================================================================
# Logistic Regression
# ==============================================================================

# Declare Theano symbolic variables
x = T.matrix("x")
y = T.vector("y")

x.tag.test_value = train_set_x
y.tag.test_value = train_set_y

# Declare theta = (w,b) = (weight, biasvector) = (weight vector, bias vector)
weight = theano.shared(
    np.zeros(num_features).astype(theano.config.floatX), name="weight")

biasvector = theano.shared(
    np.asarray(0., dtype=theano.config.floatX), name="biasvector")

hypothesis = T.nnet.sigmoid(T.dot(x, weight) + biasvector)

# The prediction threshold
prediction = hypothesis > 0.5

# Cross-entropy loss function
celf = -y * T.log(hypothesis) - (1 - y) * T.log(1 - hypothesis)

# Regularization parameter
var_lambda = 1.0

# The cost to optimize (with regularization)
cost = celf.mean() + (var_lambda / (2.0 * num_examples)) * (weight ** 2).sum()

# The gradient of the cost function wrt weight
grad_of_weight, grad_of_biasvector = T.grad(cost, [weight, biasvector])

# Learning rate
var_alpha = 0.005

# Updates to be performed during optimization
updates={weight: weight - var_alpha * grad_of_weight,
biasvector: biasvector - var_alpha * grad_of_biasvector}

# Compile expressions to function.
train = theano.function(
    inputs=[x, y],
    outputs=[prediction, celf, cost],
    updates=updates,
    name="train",
    allow_input_downcast=False)

predict = theano.function(
    inputs=[x],
    outputs=prediction,
    name="predict",
    allow_input_downcast=False)

# Error functions
error = celf.mean()

err_func = theano.function(
    inputs=[x, y],
    outputs=[error],
    name="err_func",
    allow_input_downcast=False)

# The number of times to run the loop to optimize the cost function
training_steps = 50000

# Start training the algorithm
#for i in range(training_steps):
#    pred, err, costing = train(train_set_x, train_set_y)

train_err = []
cross_err = []

for i in range(1, num_examples):
    Xtemp = train_set_x[0:i,::]
    ytemp = train_set_y[0:i]
    train(Xtemp, ytemp)
    train_err.append(err_func(Xtemp, ytemp)[0].tolist())
    cross_err.append(err_func(cross_set_x, cross_set_y)[0].tolist())

train_err = np.array(train_err, dtype=np.float32)
cross_err = np.array(cross_err, dtype=np.float32)

# Generate the learning curve
plt.figure()
plt.plot(range(1, num_examples), cross_err, 'b')
plt.plot(range(1, num_examples), train_err, 'r')
plt.xlabel('Number of Examples')
plt.ylabel('Error')
plt.show()

# Get all the predicted values
predicted_vector = predict(test_set)

# Generate the submission file
open_file_object = csv.writer(open("logist_theano_submission.csv", "wb"))
test_file_object = csv.reader(open('../../data_lab/test.csv', 'rb'))

test_file_object.next()

i = 0
for row in test_file_object:
    row.insert(0,predicted_vector[i].astype(np.uint8))
    open_file_object.writerow(row)
    i += 1


import sys
import os
from setuptools import setup

from joseph.constants import REQUIREMENTS_FILE

version = "0.0.1"

def joseph_plugins(requirements=REQUIREMENTS_FILE):
    plugins = []
    if os.path.isfile(requirements):
        with open(requirements, "r") as in_file:
            for line in in_file:
                if not line.startswith(("-", "#")):
                    plugins.append(line)

    return plugins

install_requires = [
    "aiofiles",
    "aiohttp",
    "jinja2",
    "pytimeparse",
    "ruamel.yaml",
    "ruamel.yaml.jinja2",
]

if sys.platform == "darwin":
    # Only require UVLoop on supported systems
    install_requires.append("uvloop")

extra_requires = {
    # Libraries required to run the tests
    'test': [
        "coverage==4.4.1",
        "pytest==3.2.1",
        "mock==2.0.0",
        "tox==2.8.1",
    ],
    # Py.test plugins. Separate to distinguish between test
    # libraries and their plugins
    'pytest.plugins': [
        "pytest-pep8==1.0.6",
        "pytest-cov==2.5.1",
        "pytest-asyncio==0.6.0",
        "pytest-timeout==1.2.0",
    ],
    # Optional Joseph plugins (examples as well as user installed plugins).
    'joseph.plugins': [
        "joseph-hello-world>=0.3.1",
        "joseph_web_hello_world>=0.1.5",
    ] + joseph_plugins()
}

setup(
    name="Joseph_Automation",
    version=version,
    description="A home automation platform with framework aspirations",
    author="Niek Keijzer",
    author_email="info@niekkeijzer.com",
    license="MIT",
    url="https://github.com/NiekKeijzer/joseph",
    packages=["joseph"],
    install_requires=install_requires,
    extras_require=extra_requires,
    test_suite="tests"
)


import pygame, sys
import resources.resourceManager as resM
pygame.init()

window = pygame.display.set_mode((200, 200))
background = pygame.Surface((window.get_size()))
background.fill((255, 255, 255))
image = pygame.image.load(resM.heatArrow)
image2 = pygame.image.load(resM.heatArrow)

image = image.convert()
rect = image.get_rect()

image2 = image2.convert_alpha()
rect2 = image2.get_rect()

rect2.left = rect.width + 1

i = 0
while True:
  for event in pygame.event.get():
    if event.type == 12:
      pygame.quit()
      sys.exit()

  image.set_alpha(i)
  image2.set_alpha(i)

  window.fill((255, 255, 255))
  window.blit(background, background.get_rect())
  window.blit(image, rect)
  window.blit(image2, rect2)

  if i == 255:
    i = 0
  else:
    i += 1

  pygame.display.update()
  pygame.time.Clock().tick(60)


from base import *
import dj_database_url

WSGI_APPLICATION = 'gettingstarted.wsgi.application'

# Update database configuration with $DATABASE_URL.
db_from_env = dj_database_url.config(conn_max_age=500)
DATABASES['default'].update(db_from_env)
DATABASES = {'default': dj_database_url.config(default='postgres://localhost')}

DEBUG = False


# -*- coding: utf-8 -*-

# import win32com.client as win32
import urllib.request, json
import requests
from bs4 import BeautifulSoup
from emailtemplates import get_existing_templates

def getGovDetails(postcode):

    url_base = "http://api.postcodes.io/postcodes/"
    with urllib.request.urlopen(url_base + postcode) as url:
        data = json.loads(url.read().decode())
        if data["status"] == 200:
            topdata = data["result"]
        else:
            raise KeyError("No postcode found!")

    MPurl = (
        "http://lda.data.parliament.uk/commonsmembers.json?_view=members&_pageSize=2097&_page=0&constituency.label="
        + "%20".join(topdata["parliamentary_constituency"].split(" "))
    )

    with urllib.request.urlopen(MPurl) as url:
        MPdata = json.loads(url.read().decode())

        for possibleMP in MPdata["result"]["items"]:
            MPid = (possibleMP["_about"]).split("/")[-1]
            print("Checking MP: {}".format(possibleMP["fullName"]))

            try:
                MPurl = "https://members.parliament.uk/member/{}/contact".format(MPid)
                MPemails = emailExtractor(MPurl)  # MP email (in a list)
                assert len(MPemails) > 0
                # Quick hack to only return one email
                MPemail = MPemails[0]

                myward = topdata["admin_ward"]  # User's ward
                MPname = possibleMP["fullName"]["_value"]
                print("Found correct MP: {}. Email: {} ".format(MPname, MPemail))
                break
            except:
                pass

    return {"ward": myward, "MPemail": MPemail, "MPname": MPname}

def emailExtractor(urlString):
    emailList = []
    getH = requests.get(urlString)
    h = getH.content
    soup = BeautifulSoup(h, "html.parser")
    mailtos = soup.select("a[href^=mailto]")
    for i in mailtos:
        href = i["href"]
        try:
            str1, str2 = href.split(":")
        except ValueError:
            break

        emailList.append(str2)
    return emailList

def draftEmails(myname, postcode):
    ret = getGovDetails(postcode)
    ward = ret["ward"]
    MPname = ret["MPname"]
    MPemail = ret["MPemail"]

    print(
        "Details found. You live in {} ward and your MP is {}, with email: {}".format(
            ward, MPname, MPemail
        )
    )

    # Empty list of filled templates
    filled_email_templates = []
    # Get all the empty templates from templates.py
    empty_email_templates = get_existing_templates()
    # Set the user dictionary to include the name of the person sending
    user = {"name": myname}

    for e in empty_email_templates:  # For each empty template
        if e.target is None:
            # If no defined target, use MP info to fill target fields
            e.set_target(name=MPname, email=MPemail, ward=ward)

        # ToDo : Implement setting a cc

        # Pass the dictionary containing user information to the template filler
        success = e.fill(user)  # Returns true if successfully filled

        if success:
            # Append successful templates to the list we return
            filled_email_templates.append(e)
        else:
            print("Failed to fill template, subject: {}".format(e.subject))
            pass

    return filled_email_templates


### This program presents the pictorial depiction of the impact on the weights of the features based on the different values of
### regularilization parameters. In this case we used Logistic regression.
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import Machine_Learning.CommonFunctions.commonvariables as common
import sys

colors=common.colors
weights,params = [],[]
df_wine=common.df_wine
x_train_std=common.x_train_std
x_test_std=common.x_test_std
y_train=common.y_train

fig=plt.figure()
ax=plt.subplot(111)

## Execute the Logistic Regression  using L1 regularization (lasso) with different Regularization factor.
## In this C is the factor. Penalty =l1 means L1 regularization.
for c in np.arange(-4,6):
    lr=LogisticRegression(penalty='l1',C=10**c,random_state=0)
    lr.fit(x_train_std,y_train)
    weights.append(lr.coef_[1])
    params.append(10**c)

weights = np.array(weights)
for column,color in zip(range(weights.shape[1]),colors):
    #print ("{},{}".format(column,color))
    plt.plot(params,weights[:,column],label=df_wine.columns[column+1],color=color)
    #print ("{},{}".format(params,weights[:,column]))
    #plt.show()

plt.axhline(0,color='black',linestyle='--',linewidth=3)
#plt.xlim([10**(-5),10**(-6)])
plt.xlabel('C')
plt.ylabel('Weight Coeffients')
plt.xscale('log')
plt.legend(loc='upper left')
ax.legend(loc='upper center',bbox_to_anchor=(1.38,1.03),ncol=1,fancybox=True)

plt.show()

Machine_Learning/4 - Data Preprocessing/ReguarizationPath.py


import matplotlib.pyplot as plt
import numpy as np
from math import sqrt
K =23.26
r = 12/4000
tau = 0.51
Kampli = 10
print(-(1/(4*tau**2))+(K*r*Kampli)/(tau))
A = sqrt(-(1/(4*tau**2))+(K*r*Kampli)/(tau))

x=np.linspace(0,10,1000)
p1 = plt.plot(x,
            (
                1
                -np.exp(-x/(2*tau))*np.cos(A*x)
                -(1)/(2*tau*A)*np.exp(-x/(2*tau))*np.sin(A*x)
                ),label="Angle de sortie $\\theta(t)$",lw=2)
p2 = plt.plot(x,
            (
                1+0.*x
                ),label="Angle d'entrée $\\theta_e(t)$",lw=2)

plt.grid(True, which="both", linestyle="dotted")
#plt.title("Fonctions trigonometriques")
#plt.legend([p1, p2], ["Sinus", "Cosinus"])
plt.legend(loc='lower right', fancybox=True, shadow=True, prop=dict(size=10))
plt.xlabel("Temps en $s$")
plt.ylabel("Angle en $rad$")
#plt.title("Evolution de $\theta$ en fonction du temps pour une entrée échelon de 1V")
plt.show()
fig = plt.figure()
#fig.savefig('temp.pdf', transparent=True)


# -*- coding: utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding('UTF8')

from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import numpy as np
from scipy.io import netcdf_file as netcdf
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection
import math

#lonb1 = -77.5
lonb1 = -76.5
lonb2 = -65.5
latb1 = -57
#latb1 = -65
latb2 = -15

#fig = plt.figure(figsize=[12, 7.8])
fig = plt.figure()
ax = plt.subplot(111)

#box = ax.get_position()
#ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])

cl_map = Basemap(projection='cyl', llcrnrlat = latb1, urcrnrlat = latb2, llcrnrlon = lonb1, urcrnrlon = lonb2, resolution = 'i')

cl_map.drawparallels(np.arange(-60, 51, 2.5), labels=[1,0,0,0], fontsize = 8, xoffset = .2, color = 'gray', dashes=[1,4], linewidth = .3)
cl_map.drawmeridians(np.arange(11, 360., 3), labels=[0,0,0,1], fontsize = 8, yoffset = .2, color = 'gray', dashes=[1,4], linewidth = .3)
#cl_map.drawcountries(linewidth = .5, color = '.3', zorder = 3)
#cl_map.drawcoastlines(linewidth = .5, color = '.3', zorder = 3)
#cl_map.drawstates(linewidth = .5, color = '.3', zorder = 3)

#cl_map.readshapefile('CHL_adm/CHL_adm0', 'chile_country', linewidth=0.05)
cl_map.readshapefile('CHL_adm/CHL_adm1', 'chile_regions', drawbounds=False)

patches = []
for info, shape in zip(cl_map.chile_regions_info, cl_map.chile_regions):
    patches.append( Polygon(np.array(shape), True) )
ax.add_collection(PatchCollection(patches, facecolor= '#FFCC99', edgecolor='k', linewidths=.3, zorder=2))

nc = netcdf('ETOPO1_Ice_c_gdal.grd', 'r')
dim = nc.variables['dimension'][:]
delta = nc.variables['spacing'][0]
z = nc.variables['z'][:]

lon0 = np.arange(-180,180,delta)
lat0 = np.arange(90-delta/2,-90,-delta)
zgrid = np.zeros([dim[1], dim[0]])
for y in np.arange(dim[1]):
    zgrid[y,:] = z[(dim[0]*y):(dim[0]*(y+1))]

zgrid = zgrid[6000:9000,5000:7000]
lat0 = lat0[6000:9000]
lon0 = lon0[5000:7000]

#fig = plt.figure(figsize=[12, 7.8])

lons, lats = np.meshgrid(lon0, lat0)
x, y = cl_map(lons, lats)
#cs = cl_map.contourf(x, y, zgrid, [0, 1500], colors=('w', '.8', '.6'), extend = 'both')

#cl_map.shadedrelief()

#plon = -76
#plon = -80
plon = lonb1 + 2

easter_lat = -27.15
rec = plt.Rectangle((plon - .4, easter_lat - .4), .8, .8, ec = 'k', lw = .5, fc = 'w', zorder = 3)
ax.add_patch(rec)
plt.arrow(plon - .4, easter_lat, -.6, 0, head_width = 0.3, head_length = 0.2, lw = .5, fc='k', ec='k')
#plt.text(plon, -27.95, 'I. de Pascua', size = 8, ha = 'center', va = 'center')

juan_lat = -33.66667
rec = plt.Rectangle((plon - .4, juan_lat - .4), .8, .8, ec = 'k', lw = .5, fc = 'w', zorder = 3)
ax.add_patch(rec)
plt.arrow(plon - .4, juan_lat, -.6, 0, head_width = 0.3, head_length = 0.2, lw = .5, fc='k', ec='k')
#plt.text(-78.98333, -34.44, 'J. Fdez', size = 8, ha = 'center', va = 'center')

frei_lat = -54.5
rec = plt.Rectangle((plon - .4, frei_lat - .4), .8, .8, ec = 'k', lw = .5, fc = 'w', zorder = 3)
ax.add_patch(rec)
plt.arrow(plon, frei_lat - .4, 0, -1, head_width = 0.3, head_length = 0.2, lw = .5, fc='k', ec='k')
#plt.text(plon, -51, 'Antartica', size = 8, ha = 'center', va = 'center')

#axin = inset_axes(ax, width=1.5, height=1.5, loc=8)
"""
axin = inset_axes(ax, width=1.5, height=1.5, bbox_to_anchor=(1, 1), bbox_transform=ax.transAxes)
inmap = Basemap(ax=axin,width=1200000*2,height=900000*2,projection='lcc',resolution=None,lat_1=-70,lat_0=-70,lon_0=-65.)
inmap.shadedrelief()
inmap.drawparallels(np.arange(-90, 90, 5), labels=[0,0,1,0], fontsize = 7, xoffset = .2, color = 'gray', dashes=[1,1], linewidth = .3)
inmap.drawmeridians(np.arange(0, 360., 30), labels=[0,0,0,1], fontsize = 7, yoffset = .2, color = 'gray', dashes=[1,1], linewidth = .3)
"""

#ax = plt.axes([.04, .02, .1, .2])

#ax = plt.axes([.45, .01, .1, .2])
#m = Basemap(width=1200000*2,height=900000*2,projection='lcc',resolution=None,lat_1=-70,lat_0=-70,lon_0=-65.)
#m.shadedrelief()
#m.fillcontinents(color = '.6', lake_color = 'w')
#m.drawparallels(np.arange(-90, 90, 30), labels=[1,0,0,0], fontsize = 7, xoffset = .2, color = 'gray', dashes=[1,1], linewidth = .3)
#m.drawmeridians(np.arange(0, 360., 30), labels=[0,0,0,1], fontsize = 7, yoffset = .2, color = 'gray', dashes=[1,1], linewidth = .3)

stations = np.array([
    [-18.33333, -70.33333, '#3399CC', 'Arica', 'left', 2.8, 0],
    [-18.53333, -70.31667, '#FF6600', None, 'right', 0, 0],
    [-20.53972, -70.17861, 'green', 'Iquique', 'left', 3.7, 0],
    [-22.91083, -68.2, '#FF6600', 'San Pedro de Atacama', 'left', 9.3, 0],
    [-23.45028, -70.44111, 'green', 'Antofagasta', 'right', -5.5, 0],
    #[-27.16056, -109.4272, 'green'],
    [easter_lat, plon, 'green', 'Isla de Pascua', 'right', -6.1, 0],
    [-27.26389, -70.77417, '#FF6600', 'Caldera', 'left', 3.8, 0],
    [-29.9175, -71.20111, 'green', 'La Serena', 'left', 4.6, 0],
    [-33.03833, -71.62917, 'green', 'Valparaiso', 'right', -4.7, 3],
    [-33.35, -70.29361, '#FF6600', 'Farellones', 'left', 4.6, 1.5],
    [-33.445, -70.68278, 'green', 'Santiago', 'left', 4.2, 0],
    #[-33.66667, -78.98333, '#3399CC', 'Juan Fernandez', 'right', -15, 0],
    [juan_lat, plon, '#3399CC', 'Juan Fernandez', 'right', -6.6, 1.5],
    [-34.17389, -70.77694, '#FF6600', 'Rancagua', 'right', -4.5, 0.25],
    [-34.96667, -71.23333, '#3399CC', 'Curico', 'left', 3.3, 0],
    [-35.41972, -71.66944, '#FF6600', 'Talca', 'right', -2.8, 0],
    [-36.56667, -72.03333, '#3399CC', 'Chillan', 'left', 3.4, 0],
    [-36.77833, -73.0625, 'green', 'Concepcion', 'right', -5.1, 0],
    [-36.90361, -71.41, '#FF6600', None, 'left', 1, 0],
    [-38.70306, -72.54917, '#FF6600', None, 'left', 1, 0],
    [-38.75, -72.63333, '#3399CC', 'Temuco', 'right', -3.8, 0],
    [-39.61667, -73.08333, '#3399CC', 'Valdivia', 'left', 3.8, 0],
    [-39.81389, -73.24806, '#FF6600', None, 'right', -1, 0],
    [-40.6, -73.05, '#3399CC', 'Osorno', 'left', 3.7, -1],
    [-41.435, -73.09778, 'green', 'Puerto Montt', 'right', -5.7, 0],
    [-45.59389, -72.10861, 'green', 'Coyhaique', 'right', -4.8, 0],
    [-45.91667, -71.68333, '#3399CC', 'Balmaceda', 'left', 5, 0],
    [-53.00333, -70.845, 'green', 'Punta Arenas', 'left', 5.8, 0],
    #[-62.19306, -58.9825, 'green'],
    [frei_lat, plon, 'green', 'C.M.A. Edo. Frei', 'right', -6.6, 0],
    ])

points = list()
for station in stations:

    #colors = ('#243F93', '#EC663B')
    colors = ('#3399CC', '#FF6600')

    """
    if station[2] not in colors:
        xy = []
        start = 0.75
        for ratio in (0.5, 0.5):
            x = [0] + np.cos(np.linspace(2*math.pi*start,2*math.pi*(start+ratio), 30)).tolist()
            y = [0] + np.sin(np.linspace(2*math.pi*start,2*math.pi*(start+ratio), 30)).tolist()
            xy1 = zip(x,y)
            xy.append(xy1)
            start += ratio
        for i, xyi in enumerate(xy):
            point = plt.scatter(station[1], station[0], marker=(xyi,0), s = 15, lw = .2, zorder = 4, color=colors[i])
    else:
        point = plt.scatter(station[1], station[0], s = 45, lw = .2, zorder = 4, color=station[2])
    """
    point_b = plt.scatter(station[1], station[0], s = 20, lw = .2, zorder = 4, color='black')

    if station[3]:
        if station[4] == 'left':
            text_x = lonb2 + 3
        elif station[4] == 'right':
            text_x = lonb1 - 3

        if station[2] not in colors:
            xy = []
            start = 0.75
            for ratio in (0.5, 0.5):
                x = [0] + np.cos(np.linspace(2*math.pi*start,2*math.pi*(start+ratio), 30)).tolist()
                y = [0] + np.sin(np.linspace(2*math.pi*start,2*math.pi*(start+ratio), 30)).tolist()
                xy1 = zip(x,y)
                xy.append(xy1)
                start += ratio
            for i, xyi in enumerate(xy):
                point = plt.scatter(text_x + station[5], station[0] + .2 + station[6], marker=(xyi,0), s = 180, lw = .2, zorder = 4, color=colors[i], clip_on=False)
        else:
            point = plt.scatter(text_x + station[5], station[0] + .2 + station[6], s = 180, lw = .2, zorder = 4, color=station[2], clip_on=False)

        points.append(point)

        ax.annotate(
            station[3],
            (station[1], station[0]),
            xytext=(text_x, station[0] + station[6]),
            #textcoords='offset points',
            ha=station[4],
            fontsize=10,
            bbox={'boxstyle':'square,pad=0.5','fc':'#999999', 'linewidth':0},
            arrowprops={'arrowstyle': '-', 'linewidth': .5}
            )

"""
ax.legend(tuple(points),
           ('Estacion climatologica', 'Estacion UV', 'Estacion climatologica UV'),
           scatterpoints=1,
           loc='upper left',
           ncol=1,
           fontsize=12,
           bbox_to_anchor=(1, 1)
           )
"""

ax.legend(tuple(points),
            ('Estacion climatologica', 'Estacion climatologica UV', 'Estacion UV'),
            scatterpoints=1,
            loc='upper center',
            ncol=1,
            fontsize=12,
            bbox_to_anchor=(0.5, 1.07),
            fancybox=True,
            shadow=True
            )

mng = plt.get_current_fig_manager()
mng.resize(*mng.window.maxsize())
plt.show()


"""
This module contains the CodeHasher class, which hashes code objects
into values that uniquely identify those objects.

Bionic uses the CodeHasher class to detect any changes to entity
functions. When an entity function changes, the entity value may also
change, which requires Bionic to invalidate the cache and compute the
entity again.

This class is only used when versioning mode is set to "auto" or
"assisted". Since the logic doesn't cover every type of code change,
like changes to references or classes, these versioning modes are still
experimental. When we can detect all kinds of code changes, we will
make "auto" mode the default behavior.

Since Bionic uses these encodings to detect changes, any changes to
encoding will make caching backwards-incompatible. If you make any
changes to this module that can change the encoding, update
CACHE_SCHEMA_VERSION to update cache scheme.
"""

from enum import Enum
import hashlib
import inspect
import warnings

from .utils.misc import oneline

PREFIX_SEPARATOR = b"$"

class CodeHasher:
    """
    This class hashes code objects into values that uniquely identify
    those objects. If two code objects have different behavior, they
    should ideally have different hashes. However, for now, we only
    examine the function bytecode and associated constants. We will add
    more functionality like checking function references and any
    changes to class objects.

    Since this class doesn't examine the references and class objects,
    it doesn't actually achieve the ideal behavior we described above.
    """

    def __init__(self):
        self._hash = hashlib.new("md5")
        # This is used to detect circular references.
        self._object_depths_by_id = {}

    @classmethod
    def hash(cls, obj):
        hasher = cls()
        hasher._check_and_ingest(obj=obj)
        return hasher._hash.hexdigest()

    def hexdigest(self):
        return self._hash.hexdigest()

    def update(self, obj):
        return self._check_and_ingest(obj)

    def _ingest_raw_prefix_and_bytes(self, type_prefix, obj_bytes=b""):
        self._hash.update(type_prefix.value)
        self._hash.update(get_size_as_bytes(obj_bytes))
        self._hash.update(PREFIX_SEPARATOR)
        self._hash.update(obj_bytes)

    def _check_and_ingest(self, obj):
        """
        Checks for circular references before calling the _ingest
        method, which does the actual encoding.
        """
        obj_id = id(obj)

        # If the obj is already being hashed, break the circular ref by
        # analyzing the depth of the value instead.
        if obj_id in self._object_depths_by_id:
            obj = self._object_depths_by_id[obj_id]
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.CIRCULAR_REF,
                obj_bytes=str(self._object_depths_by_id[obj_id]).encode(),
            )
            return

        self._object_depths_by_id[obj_id] = len(self._object_depths_by_id)
        self._ingest(obj)
        del self._object_depths_by_id[obj_id]

    def _ingest(self, obj):
        """
        Contains the logic that analyzes the objects and encodes them
        into bytes that are added to the hash.
        """
        if isinstance(obj, bytes):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.BYTES, obj_bytes=obj
            )

        elif isinstance(obj, bytearray):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.BYTEARRAY, obj_bytes=obj
            )

        elif obj is None:
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.NONE, obj_bytes=b"None"
            )

        elif isinstance(obj, int):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.INT,
                obj_bytes=str(obj).encode(),
            )

        elif isinstance(obj, float):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.FLOAT,
                obj_bytes=str(obj).encode(),
            )

        elif isinstance(obj, str):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.STRING,
                obj_bytes=obj.encode(),
            )

        elif isinstance(obj, bool):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.BOOL,
                obj_bytes=str(obj).encode(),
            )

        elif isinstance(obj, (list, set, tuple)):
            if isinstance(obj, list):
                type_prefix = TypePrefix.LIST
            elif isinstance(obj, set):
                type_prefix = TypePrefix.SET
            else:
                type_prefix = TypePrefix.TUPLE
            obj_bytes = str(len(obj)).encode()
            self._ingest_raw_prefix_and_bytes(
                type_prefix=type_prefix,
                obj_bytes=obj_bytes,
            )
            for elem in obj:
                self._check_and_ingest(elem)

        elif isinstance(obj, dict):
            self._ingest_raw_prefix_and_bytes(
                type_prefix=TypePrefix.DICT,
                obj_bytes=str(len(obj)).encode(),
            )
            for key, elem in obj.items():
                self._check_and_ingest(key)
                self._check_and_ingest(elem)

        elif inspect.isroutine(obj):
            self._ingest_raw_prefix_and_bytes(type_prefix=TypePrefix.ROUTINE)
            self._check_and_ingest(obj.__defaults__)
            self._ingest_code(obj.__code__)

        elif inspect.iscode(obj):
            self._ingest_raw_prefix_and_bytes(type_prefix=TypePrefix.CODE)
            self._ingest_code(obj)

        else:
            # TODO: Verify that we hash all Python constant types.
            self._ingest_raw_prefix_and_bytes(type_prefix=TypePrefix.DEFAULT)
            message = oneline(
                f"""
                Found a constant {obj!r} of type {type(obj)!r} that
                Bionic doesn't know how to hash. This is most likely a
                bug in Bionic. Please raise a new issue at
                https://github.com/square/bionic/issues to let us know.
                """
            )
            warnings.warn(message)

    def _ingest_code(self, code):
        # TODO: Find references for the code and analyze references.
        self._check_and_ingest(code.co_code)

        # TODO: Maybe there is a way using which we can differentiate
        # between lambda variable names and string constants that end
        # with `.<lambda>`.
        consts = [
            const
            for const in code.co_consts
            if not (isinstance(const, str) and const.endswith(".<lambda>"))
        ]
        self._check_and_ingest(consts)

class TypePrefix(Enum):
    """
    Represents a unique value for each type that CodeHasher hashes that
    is prefixed to avoid collision between same encoded values.

    If you change the prefix of any type, or add a new prefix, it can
    change the encoding of objects. Since Bionic uses these encodings
    to detect changes, any changes to encoding will make caching
    backwards-incompatible. Update CACHE_SCHEMA_VERSION to update cache
    scheme if you change the encoding.
    """

    BYTES = b"AA"
    BYTEARRAY = b"AB"
    NONE = b"AC"
    STRING = b"AD"
    INT = b"AE"
    FLOAT = b"AF"
    BOOL = b"AG"
    LIST = b"AH"
    SET = b"AI"
    TUPLE = b"AJ"
    DICT = b"AK"
    ROUTINE = b"AL"
    CODE = b"AM"
    CIRCULAR_REF = b"AN"
    DEFAULT = b"ZZ"

def get_size_as_bytes(obj_bytes):
    assert isinstance(obj_bytes, (bytes, bytearray))
    return str(len(obj_bytes)).encode()


# https://github.com/DavidWangWood/Camera-Calibration-Python/blob/master/camera_calibration.ipynb
# https://docs.opencv.org/3.3.0/dc/dbb/tutorial_py_calibration.html
# https://docs.opencv.org/master/dc/dbb/tutorial_py_calibration.html

import numpy as np
import cv2
import glob

# termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((7 * 10, 3), np.float32)
objp[:, :2] = np.mgrid[0:10, 0:7].T.reshape(-1, 2)

# Arrays to store object points and image points from all the images.
objpoints = []  # 3d point in real world space
imgpoints = []  # 2d points in image plane.
# images = glob.glob('*.bmp')
images=glob.glob('*.bmp')
print(images)
k=0
p=0
for fname in images:
    img = cv2.imread(fname)
    img_size = (img.shape[1], img.shape[0])
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Find the chess board corners
    ret, corners = cv2.findChessboardCorners(gray, (10, 7), None)
    if ret==False:
        p+=1
    print("ominięto", p)
    # If found, add object points, image points (after refining them)
    if ret == True:
        k+=1
        objpoints.append(objp)
        corners2 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners)

        # Draw and display the corners

        kozak=cv2.drawChessboardCorners(gray, (10, 7), corners2, ret)
        cv2.imwrite(f"obrazek{k}.jpg",kozak)
        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)
        k+=1
        print(ret)
        print("przyjeto",k)
        dst = cv2.undistort(img, mtx, dist, None, mtx)

        cv2.imwrite(f'obrazek{k}.jpg', dst)
        # dst = cv2.undistort(dist, mtx, dist, None, mtx)
        # cv2.imshow('img',gray)

        cv2.waitKey(0)

# cv2.destroyAllWindows()
img=cv2.imread('img00308_r.bmp')
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)
print(ret)
dst = cv2.undistort(img, mtx, dist, None, mtx)
cv2.imwrite('undistorted.jpg', dst)

import matplotlib.pyplot as plt

# Test undistortion on an image

# Do camera calibration given object points and image points
# ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)
#
# dst = cv2.undistort(img, mtx, dist, None, mtx)
# cv2.imwrite('TEST.jpg', dst)
#
#
# # dst = cv2.cvtColor(dst, cv2.COLOR_BGR2RGB)
# # Visualize undistortion
# f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
# ax1.imshow(img)
# ax1.set_title('Original Image', fontsize=30)
# ax2.imshow(dst)
# ax2.set_title('Undistorted Image', fontsize=30)
# plt.show()


from django.urls import include, path
from rest_framework.authtoken import views
from rest_framework.routers import DefaultRouter

from .views import CommentViewSet, ReviewViewSet

router = DefaultRouter()

router.register("titles/(?P<title_id>\d+)/reviews", ReviewViewSet, basename="reviews")
router.register(
    "titles/(?P<title_id>\d+)/reviews/(?P<review_id>\d+)/comments",
    CommentViewSet,
    basename="comments",
)

urlpatterns = [
    path("api/v1/", include(router.urls)),
]


from flask import Flask,render_template,redirect,url_for,session, request, jsonify, make_response
from service.controller import bp_user as app
#디비
from service.model import dbHelper #객체 변수 땡겨옴
from service import config

#라우팅
@app.route('/')
def home() :
    #세션이 없으면 /login으로 리다이렉트
    # 만약 이 서버에 접속하는 사람이 2명이상일때도 이 조건문이 성립하는가?
    # if not 'uid' in session :
    #     return redirect(url_for('login'))

    ################################################
    # 쿠키 적용 : 응답을 이용 - 사용자의 브라우저에다가 특정정보를 남기는 것
    # ->아이디를 저장해서 로그인페이지 뜰때 자동으로 아이디가 보이게 하기.
    # 응답 객체를 생성한다.

    # 쿠키 세팅
    resp=make_response(render_template('index.html',config=config)) #render template 반환할때 쿠키 세팅해서 응답하기.
    resp.set_cookie('uid', session['uid'])
    return resp

#~/url/login ->url_for로 구현되기때문에 굳이 이렇게 url이 바뀌어도 괜찮게 됨..!!!
@app.route('/login',methods=['POST','GET'])
def login():
    if request.method=='GET':
        #쿠키 획득 : get방식일때
        uid=request.cookies.get('uid')
        #쿠키가 없을 경우 : None으로 나오기 때문에 기본값 처리
        if not uid: #uid==None
            uid=''
        config.cookie_uid=uid
        return render_template('login.html',config=config)
    else :
        uid=request.form['uid'] #post 방식으로 요청
        upw=request.form['upw']
        row=dbHelper.loginSql(uid=uid,upw=upw) #디비쿼리수행
        # false :[],(),{},0,""
        # row ->dit =>{..}
        # row에 uid upw name 저장
        if row :
            #세션처리(필요한 정보를 세션으로 저장한다) -서버 메모리에 저장
            session['uid']= uid
            session['name']=row['name']
            #세션종료
            return redirect(url_for('userbp.home'))
        else :
            return render_template('common/alert.2.html',msg='회원이 아닙니다')

    #경고창 ->되돌아가기

#로그아웃
#~/user/logout
@app.route('/logout')
def logout():
    if not 'uid' in session :
        return redirect(url_for('userbp.login'))
    #세션 종료
    print(session)
    if 'uid' in session:
        session.pop('uid',None)
    if 'name' in session:
        session.pop('name',None)
        print("*"*50)
    print('세션제거후 :',session)
    #페이지 요청을 리다이렉트 ->홈페이지
    return redirect(url_for('userbp.home'))


import os, sys, argparse, re, glob
from utils import *

parser = argparse.ArgumentParser()
parser.add_argument('--numRuns', type=int, help='Define number of runs to be analysed.', required=True)
parser.add_argument('--hepd', action='store_true', help='Analyse HEPD data.')
parser.add_argument('--hepp', action='store_true', help='Analyse HEPP data.')
parser.add_argument('--merge', action='store_true', help='Merge all runs.')
parser.add_argument('--ana', action='store_true', help='Analyse all runs.')
parser.add_argument('-q','--quiet', action='store_true', help='Run without printouts.')
args,_=parser.parse_known_args()

runs = args.numRuns
os.system('source /opt/exp_software/limadou/set_env_standalone.sh')

datapaths = []
if args.hepd:
    datapaths = glob.glob('/storage/gpfs_data/limadou/data/flight_data/L3h5/*.h5')
# select HEPP data from 22-26.02.2019 (solar quiet period)
elif args.hepp:
    datapaths = glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190222*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190223*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190224*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190225*.h5')
    datapaths += glob.glob('/storage/gpfs_data/limadou/data/cses_data/HEPP_LEOS/*HEP_1*20190226*.h5')

if not args.merge and not args.ana:

    if len(datapaths) < runs:
        print("Only {} files available for reading. ".format(len(datapaths)))
        runs = len(datapaths)

    for irun,run in enumerate(datapaths):
        if irun>(runs-1):
            break
        outfile = home()+"/root/"+(os.path.split(run)[1]).replace("h5","root")

        print("Test if output exists: ", outfile)
        if os.path.isfile(outfile):
            print("Output root file already exists... \n read in the next file. ")
            runs=runs+1
        else:
            cmd='python3 python/readH5.py --inputFile '+str(run)
            if not args.quiet:
                cmd+=' --debug'
            if args.hepd:
                cmd+=' --data hepd'
            elif args.hepp:
                cmd+=' --data hepp'
            print(cmd)
            os.system(cmd)

# run analysis on single merged root file
elif args.ana:
    mge = 'root/all_hepd_'+str(runs)+'runs.root'
    print("Run analysis on single file: ", str(mge))
    if args.hepp:
        mge = 'root/all_hepp_'+str(runs)+'runs.root'

    # open root file
    runana='python3 python/analyseRoot.py --inputFile '+str(mge)
    os.system(runana)

# merge all root files
elif args.merge:
    mge=''
    runList=[]

    if args.hepd:
        # write
        mge = 'root/all_hepd_'+str(runs)+'runs.root'
        runList = glob.glob('root/CSES_HEP_DDD_*.root')
    elif args.hepp:
        mge = 'root/all_hepp_'+str(runs)+'runs.root'
        runList = glob.glob('root/CSES_01_HEP_1_*.root')

    if len(runList) < runs:
        print("Only {} files available for merge. ".format(len(runList)))
        runs = len(runList)
        if args.hepd:
            mge = 'root/all_hepd_'+str(runs)+'runs.root'
        elif args.hepp:
            mge = 'root/all_hepp_'+str(runs)+'runs.root'

    print("Merge files in: ", mge)
    merge(mge, runList, runs)


#Queue implementation using List
stockPrice=[]
stockPrice.insert(0,100)
stockPrice.insert(0,131)
stockPrice.insert(0,155)
#Since i am inserting in 0th position everytime, the previous datas are shifted to right
print(stockPrice)
print(stockPrice.pop())

#Instead of list as queue, we can use deque module

from collections import deque
q=deque()
#deque can be used as both queue and stack. appendleft for queu whereas append for stack
q.appendleft(100)
q.appendleft(20)
q.appendleft(250)
print(q.pop())


# Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import Callable, Dict, Tuple

import torch
from torch import nn

from flash.core.data.io.input import DataKeys
from flash.core.data.transforms import ApplyToKeys, kornia_collate, merge_transforms
from flash.core.utilities.imports import _ALBUMENTATIONS_AVAILABLE, _KORNIA_AVAILABLE, _TORCHVISION_AVAILABLE, requires

if _KORNIA_AVAILABLE:
    import kornia as K

if _TORCHVISION_AVAILABLE:
    import torchvision
    from torchvision import transforms as T

if _ALBUMENTATIONS_AVAILABLE:
    import albumentations

class AlbumentationsAdapter(torch.nn.Module):
    @requires("albumentations")
    def __init__(self, transform):
        super().__init__()
        if not isinstance(transform, list):
            transform = [transform]
        self.transform = albumentations.Compose(transform)

    def forward(self, x):
        return torch.from_numpy(self.transform(image=x.numpy())["image"])

def default_transforms(image_size: Tuple[int, int]) -> Dict[str, Callable]:
    """The default transforms for image classification: resize the image, convert the image and target to a tensor,
    collate the batch, and apply normalization."""
    if _KORNIA_AVAILABLE and os.getenv("FLASH_TESTING", "0") != "1":
        #  Better approach as all transforms are applied on tensor directly
        return {
            "to_tensor_transform": nn.Sequential(
                ApplyToKeys(DataKeys.INPUT, torchvision.transforms.ToTensor()),
                ApplyToKeys(DataKeys.TARGET, torch.as_tensor),
            ),
            "post_tensor_transform": ApplyToKeys(
                DataKeys.INPUT,
                K.geometry.Resize(image_size),
            ),
            "collate": kornia_collate,
            "per_batch_transform_on_device": ApplyToKeys(
                DataKeys.INPUT,
                K.augmentation.Normalize(torch.tensor([0.485, 0.456, 0.406]), torch.tensor([0.229, 0.224, 0.225])),
            ),
        }
    return {
        "pre_tensor_transform": ApplyToKeys(DataKeys.INPUT, T.Resize(image_size)),
        "to_tensor_transform": nn.Sequential(
            ApplyToKeys(DataKeys.INPUT, torchvision.transforms.ToTensor()),
            ApplyToKeys(DataKeys.TARGET, torch.as_tensor),
        ),
        "post_tensor_transform": ApplyToKeys(
            DataKeys.INPUT,
            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ),
        "collate": kornia_collate,
    }

def train_default_transforms(image_size: Tuple[int, int]) -> Dict[str, Callable]:
    """During training, we apply the default transforms with additional ``RandomHorizontalFlip``."""
    if _KORNIA_AVAILABLE and os.getenv("FLASH_TESTING", "0") != "1":
        #  Better approach as all transforms are applied on tensor directly
        transforms = {
            "post_tensor_transform": ApplyToKeys(DataKeys.INPUT, K.augmentation.RandomHorizontalFlip()),
        }
    else:
        transforms = {"pre_tensor_transform": ApplyToKeys(DataKeys.INPUT, T.RandomHorizontalFlip())}
    return merge_transforms(default_transforms(image_size), transforms)


import http
import os

from resource_api.common.constants import Constants
from resource_api.common.dynamo import DynamoDB
from resource_api.common.helpers import response

from resource_api.modify_resource.main.RequestHandler import RequestHandler

_dynamodb = None

def handler(event, context):
    """
    Handler method for modify resource function.
    """
    if event is None:
        return response(http.HTTPStatus.BAD_REQUEST, Constants.error_insufficient_parameters())
    if event is None or Constants.event_body() not in event or Constants.event_http_method() not in event:
        return response(http.HTTPStatus.BAD_REQUEST, Constants.error_insufficient_parameters())
    if event[Constants.event_body()] is None or len(event[Constants.event_body()]) is 0:
        return response(http.HTTPStatus.BAD_REQUEST, Constants.error_insufficient_parameters())

    global _dynamodb
    if _dynamodb is None:
        try:
            ddb = DynamoDB()
            _dynamodb = ddb.connect(os.environ[Constants.env_var_region()])
        except Exception as e:
            return response(http.HTTPStatus.INTERNAL_SERVER_ERROR, str(e))

    try:
        request_handler = RequestHandler(_dynamodb)
    except Exception as e:
        return response(http.HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
    return request_handler.handler(event, context)

def clear_dynamodb():
    """
    Clear the global dynamodb instance.
    """
    globals()['_dynamodb'] = None


## User Agent: 유저가 사람임을 인식 시키는 코드
## User Agent 가 없으면 웹 사이트에서 침입으로 인색해 차단할 가능성이 있음

# User Agent string을 검색해서 자신의 user agent를 가져옴
import requests
url = 'http://nadocoding.tistory.com'

# user agent를 입력
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36'}

# requests.get에 url뿐 아니라 user agent를 넣어줘서 실제 검색할 때와 같은 조건으로 웹 스크래핑을 해옴
res = requests.get(url, headers = headers)
res.raise_for_status()

with open('requests_test2.html', 'w', encoding='utf8') as f:
    f.write(res.text)


import os
from setuptools import setup

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

classifiers = """\
Development Status :: alpha
Environment :: Console
Intended Audience :: Science/Research
Intended Audience :: Developers
License :: GPL
Operating System :: OS Independent
Programming Language :: Python
Topic :: Scientific/Engineering
Topic :: Software Development :: Libraries :: Python Modules
"""

setup(
    name = "PyFVCOM",
    version = "1.0",
    author = "Pierre Cazenave",
    author_email = "pica@pml.ac.uk",
    description = ("PyFVCOM is a collection of various tools and utilities which can be used to extract, analyse and plot input and output files from FVCOM."),
    license = "MIT",
    keywords = "fvcom",
    platforms = "any",
    url = "https://gitlab.ecosystem-modelling.pml.ac.uk/pica/PyFVCOM",
    packages=['PyFVCOM'],
    long_description=read('README.md'),
    classifiers=classifiers
)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('login', '0002_appt'),
    ]

    operations = [
        migrations.RenameField(
            model_name='appt',
            old_name='doctor_name',
            new_name='prescription',
        ),
        migrations.RemoveField(
            model_name='appt',
            name='patient_user',
        ),
        migrations.RemoveField(
            model_name='appt',
            name='uname',
        ),
        migrations.AddField(
            model_name='appt',
            name='doctor',
            field=models.ForeignKey(related_name='doctor_user', to=settings.AUTH_USER_MODEL, null=True),
        ),
        migrations.AddField(
            model_name='appt',
            name='patient',
            field=models.ForeignKey(related_name='patient_user', to=settings.AUTH_USER_MODEL, null=True),
        ),
    ]


# -*- coding: utf-8 -*-

import jieba
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.cluster import KMeans
import pandas as pd
import random
import numpy as np

class KmeansClustering():
    def __init__(self, stopwords_path=None):
        self.stopwords = self.load_stopwords(stopwords_path)
        self.vectorizer = CountVectorizer()
        self.transformer = TfidfTransformer()

    def load_stopwords(self, stopwords=None):
        """
        加载停用词
        :param stopwords:
        :return:
        """
        if stopwords:
            with open(stopwords, 'r', encoding='utf-8') as f:
                return [line.strip() for line in f]
        else:
            return []

    def preprocess_data(self, contents):
        """
        文本预处理，每行一个文本
        :param corpus_path:
        :return:
        """
        corpus = []

        for line in contents:
            corpus.append(' '.join([word for word in jieba.lcut(line.strip()) if word not in self.stopwords]))
        return corpus

    def get_text_tfidf_matrix(self, corpus):
        """
        获取tfidf矩阵
        :param corpus:
        :return:
        """
        tfidf = self.transformer.fit_transform(self.vectorizer.fit_transform(corpus))

        # 获取词袋中所有词语
        # words = self.vectorizer.get_feature_names()

        # 获取tfidf矩阵中权重
        from scipy.sparse import coo_matrix
        weights = coo_matrix(tfidf, dtype=np.float64).toarray()

        return weights

    def kmeans(self, corpus_path, n_clusters=5):
        """
        KMeans文本聚类
        :param corpus_path: 语料路径（每行一篇）,文章id从0开始
        :param n_clusters: ：聚类类别数目
        :return: {cluster_id1:[text_id1, text_id2]}
        """
        corpus = self.preprocess_data(corpus_path)
        weights = self.get_text_tfidf_matrix(corpus)

        clf = KMeans(n_clusters=n_clusters)

        # clf.fit(weights)

        y = clf.fit_predict(weights)

        # 中心点
        # centers = clf.cluster_centers_

        # 用来评估簇的个数是否合适,距离约小说明簇分得越好,选取临界点的簇的个数
        # score = clf.inertia_

        # 每个样本所属的簇

        semi["class_label"] = y

        result = {}
        for text_idx, label_idx in enumerate(y):
            if label_idx not in result:
                result[label_idx] = [text_idx]
            else:
                result[label_idx].append(text_idx)
        return result

if __name__ == '__main__':

    random.seed(42)
    np.random.seed(42)

    Kmeans = KmeansClustering(stopwords_path='../data/stop_words.txt')
    semi = pd.read_csv("../data/unlabeled_data.csv")
    result = Kmeans.kmeans(semi["content"], n_clusters=10)
    semi.to_csv("semi.csv", index=False)
    print(result)


import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torch
import torchaudio
import torchaudio.transforms as transforms
import torchvision.transforms as tvt
import torch.optim as optim

import models
from tqdm import tqdm
import os
import random
import math
import librosa

import argparse
import mat_transforms

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--encoder', type=str, default=None,
                    help='Encoder path')
parser.add_argument('--decoder', type=str, default=None,
                    help='Decoder path')
parser.add_argument('--style', type=str, default=None,
                    help='Style image path')
parser.add_argument('--content', type=str, default=None,
                    help='Content image path')
parser.add_argument('--x', type=int, default=1,
                    help='Number of AE layers to use')

args = parser.parse_args()

def load_audio(fname, transform):
    # output, _ = torchaudio.load(fname, normalization=True)

    # output = transforms.MelSpectrogram(sample_rate=16000,
    #                                    n_fft=400, win_length=400,
    #                                    hop_length=160, n_mels=128)(output)
    y, _ = librosa.load(fname, 16000)
    stft_y = librosa.core.stft(y, n_fft=400, hop_length=160, win_length=400)
    mag = np.abs(stft_y)
    ang = np.angle(stft_y)
    D = mag**2
    output = librosa.feature.melspectrogram(S=D, sr=16000)

    output = torch.Tensor(output)
    output = output.unsqueeze(0)

    output = output[:, :, :1000]
    ang = ang[:, :1000]
    if transform is not None:
        output = transform(output)

    # output = output.unsqueeze(0)  # [1, 128, 1000]
    return output, ang

transform = tvt.Compose(
        [transforms.AmplitudeToDB(stype='power', top_db=None),
         tvt.Normalize(mean=[-38.39992], std=[13.462255])])

reverse_normalize = tvt.Normalize(
    mean=[38.39992/13.462255],
    std=[1./13.462255])

num_layers = args.x

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Assuming that we are on a CUDA machine, this should print a CUDA device:

print(device)

encoder = models.encoder(x=num_layers, pretrained_path=args.encoder).to(device)
decoder = models.decoder(x=num_layers, pretrained_path=args.decoder).to(device)

encoder.train(False)
decoder.train(False)

content_audio, content_ang = load_audio(args.content, transform)
content_audio = content_audio.to(device)
style_audio, _ = load_audio(args.style, transform)
style_audio = style_audio.to(device)

z_content, maxpool_content = encoder(content_audio)  # (1, C, W)
z_style, _ = encoder(style_audio)  # (1, C, W)

n_channels = z_content.size()[1]  # C
n_1 = z_content.size()[2]  # W

z_content = z_content.squeeze(0) # .view([n_channels, -1])  # (C, HW)
z_style = z_style.squeeze(0) # .view([n_channels, -1])  # (C, HW)

white_content = mat_transforms.whitening(z_content.cpu().detach().numpy())  # (C, HW)
color_content = mat_transforms.colouring(z_style.cpu().detach().numpy(), white_content)  # (C, HW)

# alpha = 0.6
# color_content = alpha*color_content + (1.-alpha)*z_content.cpu().detach().numpy()

color_content = torch.Tensor(color_content) # tvt.ToTensor()(color_content)
# color_content = color_content.view([n_channels, n_1, n_2]) # (C, H, W)
color_content = color_content.unsqueeze(0) # (1, C, W)

inputs_hat = decoder(color_content.to(device), maxpool_content)

new_audio = inputs_hat # .squeeze(0) # (1, C, W)
new_audio = reverse_normalize(new_audio) # (1, C, W)
new_audio = new_audio[0] # take only 1 channel (C, W)

new_audio = np.maximum(np.minimum(new_audio.cpu().detach().numpy(), 0.0), -80.0)
power_spectro = librosa.core.db_to_power(new_audio, ref=1.0)

inv_power = librosa.feature.inverse.mel_to_stft(power_spectro, sr=16000, n_fft=400, power=2.0)

full_stft = inv_power*np.exp(1j*content_ang)
y_new = librosa.core.istft(full_stft, hop_length=160, win_length=400)
# y_new = librosa.feature.inverse.mel_to_audio(power_spectro, sr=16000,
#                                              n_fft=400, hop_length=160,
#                                              win_length=400, n_iter=32) # it will take sqrt

librosa.output.write_wav("processed.wav", y_new, 16000)


import json
import os
import sys
import uuid
import shutil
import zipfile
import re
from dataclasses import dataclass, asdict, field

import requests
from dacite import from_dict
from typing import List, Dict

from judger_worker import judge
from run_utils import md5
from config import BASE_URL

@dataclass
class ClientTask:
    name: str
    config: dict = field(default_factory=dict)

@dataclass
class FileTree:
    files: List[str] = field(default_factory=list)
    folders: Dict[str, 'FileTree'] = None

@dataclass
class ClientConfig:
    tasks: List[ClientTask] = field(default_factory=list)
    files: FileTree = None

def parse_file_tree(root: dict = None) -> FileTree:
    if root is None:
        return None
    files = root['files']
    folders = None
    if root['folders'] is not None:
        folders = dict()
        for k, v in root['folders'].items():
            folders[k] = parse_file_tree(v)
    return FileTree(files=files, folders=folders)

def parse_config(path: str):
    with open(path, 'r') as fin:
        config_data = json.loads(fin.read())
    tasks = [from_dict(data_class=ClientTask, data=task) for task in config_data['tasks']]
    files = parse_file_tree(config_data['files'])
    ret = ClientConfig(tasks=tasks, files=files)
    # print(json.dumps(asdict(ret), indent=2))
    return ret

def load_file_tree(tree: FileTree, root: str = None) -> dict:
    if tree is None:
        return None
    ret = dict()
    for file in tree.files:
        path = os.path.join(root, file)
        if os.path.exists(path):
            with open(path, 'r') as fin:
                ret[file] = fin.read()
        else:
            print('ENOENT', path)
    if tree.folders is not None:
        for f, t in tree.folders.items():
            ret[f] = load_file_tree(t, os.path.join(root, f))
    return ret

def load_user_files(tree: FileTree, root_path: str) -> dict:
    path = os.path.join(root_path, 'src')
    users = os.listdir(path)
    ret = dict()
    for user in users:
        ret[user] = load_file_tree(tree, os.path.join(path, user))
    return ret

def load_test_cases(root: str):
    case_root = os.path.join(root, 'std')
    temp_root = os.path.join(root, 'tmp')
    if os.path.exists(temp_root):
        shutil.rmtree(temp_root)
    os.makedirs(temp_root)
    sub_task_folders = [os.path.join(case_root, case) for case in os.listdir(case_root)]
    tasks = {'type': 'io'}
    sub_tasks = list()
    sub_task_id = 1
    for folder in sub_task_folders:
        if not os.path.isdir(folder):
            continue
        files = [os.path.join(folder, case) for case in os.listdir(folder)]
        filter(lambda f_str: f_str.endswith('.in'), files)
        cases = []
        s = {
            'sub_task_id': sub_task_id,
            'type': 'mul',
            'score': 0,
            'cases': None
        }
        case_id = 1
        in_files = list()
        for f in files:
            if not os.path.isfile(f):
                continue
            name = int(str(f.split(os.sep)[-1]).split('.')[0])
            ext = f.split('.')[-1]
            if ext == 'in':
                in_files.append(name)
        in_files.sort()
        # print(in_files)
        for f in in_files:
            input_path = os.path.join(folder, '%d.in' % f)
            output_path = os.path.join(folder, '%d.out' % f)
            if not os.path.isfile(output_path):
                in_files.remove(f)
                continue
            in_name = '%d_%d.in' % (sub_task_id, case_id)
            out_name = '%d_%d.out' % (sub_task_id, case_id)
            temp_in = os.path.join(temp_root, in_name)
            temp_out = os.path.join(temp_root, out_name)
            shutil.copyfile(input_path, temp_in)
            with open(output_path, 'r') as fin:
                output_str = '\n'.join([
                    l.rstrip() for l in fin.read().
                        replace('\r\n', '\n').
                        replace('\r', '\n').
                        split('\n')])
                with open(temp_out, 'w') as fout:
                    fout.write(output_str)
                    output_md5 = md5(output_str)
            cases.append({
                'case_id': case_id,
                'input_size': os.path.getsize(temp_in),
                'input_name': in_name,
                'output_size': os.path.getsize(temp_out),
                'output_name': out_name,
                'output_md5': output_md5
            })
            case_id += 1
        if len(cases) > 0:
            s['cases'] = cases
            sub_tasks.append(s)
            sub_task_id += 1
    score_per_sub_task = int(100 / len(sub_tasks))
    for t in sub_tasks:
        t['score'] = score_per_sub_task
    tasks['sub_tasks'] = sub_tasks
    with open(os.path.join(temp_root, 'tasks.json'), 'w') as task_out:
        task_out.write(json.dumps(tasks, indent=2))
    return tasks

def build_task_config(config: ClientConfig, root: str):
    problem_id = str(uuid.uuid4())
    operations = list()
    op_config = dict()
    load_test_cases(root)
    os.rename(os.path.join(root, 'tmp'), os.path.join(root, problem_id))
    for task in config.tasks:
        operations.append(task.name)
        op_config[task.name] = task.config
    return problem_id, operations, op_config

def zip_dir(dir_path, output_file_path):
    zip_file = zipfile.ZipFile(output_file_path, "w", zipfile.ZIP_DEFLATED)
    for path, dir_names, filenames in os.walk(dir_path):
        file_path = path.replace(dir_path, '')
        for filename in filenames:
            zip_file.write(os.path.join(path, filename), os.path.join(file_path, filename))
    zip_file.close()
    return output_file_path

def run_users(config: ClientConfig, users: dict, root: str):
    problem_id, operations, config = build_task_config(config, root)
    # print(task_id, problem_id, operations)
    # print(json.dumps(config, indent=2))
    messages = dict()
    user_list = list()
    zip_file = zip_dir(os.path.join(root, problem_id), os.path.join(root, '%s.zip' % problem_id))
    print('Transferring test cases...')
    files = {'file': open(zip_file, 'rb')}
    transfer_case = requests.post(BASE_URL + '/', files=files)
    if os.path.exists(os.path.join(root, problem_id)):
        shutil.rmtree(os.path.join(root, problem_id))
    if os.path.exists(zip_file):
        if not files['file'].closed:
            files['file'].close()
        os.remove(zip_file)
    if transfer_case.content.decode() != problem_id:
        problem_id = transfer_case.content.decode()
    for user, files in users.items():
        task_id = str(uuid.uuid4())
        user_list.append(user)
        messages[user] = judge.send(task_id=task_id,
                                    problem_id=problem_id,
                                    files=files,
                                    operations=operations,
                                    config=config)
        print('Judging', user)
    results = dict()
    for user in user_list:
        print('Retrieving results of ' + user + '...')
        result = messages[user].get_result(block=True, timeout=60*1000)
        results[user] = result
    print('Cleaning up...')
    requests.post(BASE_URL + '/clean_up', data={'problem_id': problem_id})
    # print(json.dumps(results, indent=2))
    return results

if __name__ == '__main__':
    print(os.getcwd())
    if len(sys.argv) > 1 and os.path.exists(sys.argv[1]):
        os.chdir(sys.argv[1])
    print(os.getcwd())
    config = parse_config('config.json')
    users = load_user_files(config.files, os.getcwd())
    # print(json.dumps(users, indent=2))
    # print(json.dumps([asdict(task) for task in config.tasks], indent=2))
    results = run_users(config, users, os.getcwd())
    report_path = os.path.join(os.getcwd(), 'report')
    if os.path.exists(report_path):
        shutil.rmtree(report_path)
    os.makedirs(report_path)
    for user, result in results.items():
        user_id = user
        if result['results'].__contains__('compile'):
            compile_res = result['results']['compile']
            compile_result = 'success' if compile_res['success'] else 'fail'
            compile_error = compile_res['info']['stderr']
        else:
            compile_error = ''
            compile_result = 'Ignored'
        if result['results'].__contains__('run'):
            run_res = result['results']['run']
            run_result = 'success' if run_res['success'] else 'fail'
            score = run_res['info']['score']
            output_run = True
        else:
            output_run = False
            run_result = 'ignored'
            score = 0

        with open(os.path.join(os.sep.join(os.path.realpath(__file__).split(os.sep)[:-1]), 'template.html'), 'r') as t_in:
            template = t_in.read()
        template = template.replace('{{compile_result}}', compile_result) \
            .replace('{{compile_error}}', compile_error) \
            .replace('{{user_id}}', user_id) \
            .replace('{{run_result}}', run_result) \
            .replace('{{score}}', str(score))
        reg = r'^\s*{{foreach case \|(.+)}}\s*$'
        case_match = re.search(reg, template, re.M | re.I)
        if case_match:
            rep = case_match.group(0)
            # print(rep)
            run_template = ''
            if output_run:
                fields = [s.strip() for s in case_match.group(1).split('|')]
                cases = []
                for sub_task in run_res['info']['sub_tasks']:
                    for c in sub_task['cases']:
                        c['sub_task_id'] = sub_task['sub_task_id']
                        c['type'] = sub_task['type']
                        c['score'] = sub_task['score']
                        cases.append(c)
                for c in cases:
                    line = '
'
                    for f in fields:
                        line += '' + str(c[str(f)]) + '	'
                    line += '
'
                    run_template += line
            template = template.replace(rep, run_template)
        with open(os.path.join(report_path, '%s.html' % user), 'w') as t_out:
            t_out.write(template)
            print('User {} exported.'.format(user))


'''
Created on Aug 17, 2016

@author: jvazquez
'''
import logging
from os.path import abspath, join, dirname

from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
from flask import Blueprint, g, jsonify, request, render_template
from flasgger import swag_from
from flask_jwt import jwt_required

from resource.api.allocation.allocation_period_response import \
    allocation_period_response
from resource.api.allocation.model import AllocationPeriod, Status, FREE
from resource.api.allocation.scheduler_layer import Schedule
from utils.helpers.response_creator.simple_creator import simplest
from utils.helpers.response_creator.http_statuses import CREATED, ERROR, OK
from resource.api.allocation.scheduler_layer import Schedule
from utils.helpers.acl.decorators import access_to_resource

template_folder = join(abspath(dirname(__file__)), "templates")
static_folder = join(abspath(dirname(__file__)), "static")
allocation_module = Blueprint(__name__, "allocation",
                              template_folder=template_folder,
                              static_url_path="/static/allocation",
                              static_folder=static_folder)
logger = logging.getLogger(__name__)
logger.debug("Template folder: {}".format(template_folder))

@allocation_module.route("/api/allocation/", methods=["POST"])
@jwt_required(401)
@access_to_resource(request, "allocation.create")
@swag_from("create.yml")
def create():
    scheduler = Schedule()
    data = request.get_json()
    errors = []
    alloc_list = []
    for single_data in data:
        start_time = parse(single_data["start_time"])
        end_time = parse(single_data["end_time"])
        resource_id = single_data["resource_id"]
        validation = scheduler.validator.are_dates_sane(start_time, end_time)
        if validation:
            single_data = scheduler.sanitize_data(single_data)
            alloc_id = scheduler.create(single_data, errors)
            if alloc_id:
                alloc_list.append(alloc_id)
        else:
            errors.append(single_data)

    if errors:
        # TODO: Modificar el error porque no corresponde!
        response = simplest("custom", 417, payload=None,
                            rejected_fields=errors,
                            callback=allocation_period_response)
    else:
        if not alloc_list:
            logger.error("There was an error when save the allocation for this resource: {}".format(resource_id))
            logger.error("This user did the allocation with error: {}".format(single_data["user_id"]))
        response = simplest("create", CREATED,
                            payload=alloc_list, rejected_fields=errors,
                            normal_mode=False)

    return jsonify(response), OK

@allocation_module.route("/api/allocation/<int:allocation_id>/",
                         methods=["PATCH"])
@jwt_required(401)
@access_to_resource(request, "allocation.update")
@swag_from("update.yml")
def update(allocation_id):
    errors = []
    scheduler = Schedule()
    data = request.get_json()
    data = scheduler.sanitize_data(data)
    validation = scheduler.validator.are_dates_sane(data["start_time"], data["end_time"])
    if validation:
        scheduler.update(allocation_id, data, errors)
    else:
        errors.append(data)

    if errors:
        # TODO: Modificar el error porque no corresponde!
        response = simplest("custom", 417, payload=None,
                            rejected_fields=errors,
                            callback=allocation_period_response)
    else:
        response = simplest("update", OK)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/<int:allocation_id>/",
                         methods=["GET"])
@jwt_required(401)
@access_to_resource(request, "allocation.get")
@swag_from("get.yml")
def get(allocation_id):
    scheduler = Schedule()
    allocation = scheduler.get_alloc_by_id(allocation_id)
    response = simplest("get", OK, allocation)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/all/<int:user_id>", methods=["GET"])
@jwt_required(401)
@access_to_resource(request, "allocation.get_all")
@swag_from("get_all.yml")
def get_all(user_id):
    scheduler = Schedule()
    allocation_list = scheduler.get_alloc_by_user_id(user_id)
    response = simplest("get_all", OK, allocation_list)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/all_resource/<string:resource_id>", methods=["GET"])
@jwt_required(401)
@access_to_resource(request, "allocation.get_all")
@swag_from("get_all_resource.yml")
def get_all_by_resource(resource_id):
    scheduler = Schedule()
    allocation_list = scheduler.get_alloc_by_resource_id(resource_id)
    response = simplest("get_all", OK, allocation_list)
    return jsonify(response), OK

@allocation_module.route("/api/allocation/<int:period_id>/",
                         methods=["DELETE"])
@jwt_required(401)
@access_to_resource(request, "allocation.delete")
@swag_from("delete.yml")
def delete(period_id):
    scheduler = Schedule()
    errors = []
    scheduler.delete_period(period_id, errors)
    if errors:
        # TODO: Modificar el error porque no corresponde!
        response = simplest("custom", 417, payload=None,
                            rejected_fields=errors,
                            callback=allocation_period_response)
    else:
        response = simplest("delete", OK, None)
    return jsonify(response), OK


import numpy as np
import pandas as pd
import os
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.datasets import load_iris

os.chdir('/Users/alexanderchia/hc')

np.random.seed(0)

iris = load_iris()

df = pd.DataFrame(iris.data, columns = iris.feature_names)

df['species'] = pd.Categorical.from_codes(iris.target, iris.target_names)
df['target'] = iris.target

df['is_train'] = np.random.uniform(0, 1, len(df)) <= 0.75

train = df[df['is_train'] == True]
test = df[df['is_train'] == False]

features = df.columns[:4]

clf = RandomForestClassifier()

clf.fit(train[features], train['target'])

clf.predict(test[features])

probs = clf.predict_proba(test[features])
print(probs)

preds = clf.predict(test[features])

crosstab_ = pd.crosstab(test['target'], preds, rownames=['ACTUAL'], colnames=['PREDICTED'])
print(crosstab_)