"""add column scheduledstop_log.interpolated_arrival_flag

Revision ID: 4f5fd07ef2b9
Revises:
Create Date: 2019-07-17 09:46:42.580532

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '4f5fd07ef2b9'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    op.add_column('scheduledstop_log', sa.Column('interpolated_arrival_flag', sa.Boolean))

def downgrade():
    op.drop_column('scheduledstop_log', 'interpolated_arrival_flag')


from flask import Flask
from flask import request
from flask import render_template

import calculation as Distance

app = Flask(__name__)

@app.route('/')
def my_form():
    return render_template("basic-view.html")

@app.route('/', methods=['POST'])
def my_form_post():
    point_A_text = request.form['point-A']
    point_B_text = request.form['point-B']

    point_A_lat, point_A_lon = float(point_A_text.split(',')[0]), float(point_A_text.split(',')[1])
    point_B_lat, point_B_lon = float(point_B_text.split(',')[0]), float(point_B_text.split(',')[1])

    euclid_distance = Distance.euclidean_distance(point_A_lat, point_A_lon, point_B_lat, point_B_lon)
    greatcircle_distance = Distance.great_circle_distance(point_A_lat, point_A_lon, point_B_lat, point_B_lon)
    vincenty_distance = Distance.vincenty_distance(point_A_lat, point_A_lon, point_B_lat, point_B_lon)

    print (euclid_distance)
    print (greatcircle_distance)
    print (vincenty_distance)

    euclid_error, greatcircle_error  = Distance.compute_percentage_error(euclid_distance, greatcircle_distance, vincenty_distance)
    return render_template('results.html', euclid_distance = euclid_distance, euclid_error = euclid_error, greatcircle_distance = greatcircle_distance, greatcircle_error = greatcircle_error, vincenty_distance = vincenty_distance)

if __name__ == '__main__':
    app.run()


#python

#django
from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.conf import settings
from django.conf.urls.static import static

#local
from schedule.periods import Year, Month, Week, Day

try:
    from django.conf.urls import patterns, url
except ImportError:
    from django.conf.urls.defaults import patterns, url
from django.views.generic.list import ListView
from schedule.models import Calendar
from schedule.feeds import UpcomingEventsFeed
from schedule.feeds import CalendarICalendar
from schedule.periods import Year, Month, Week, Day
from schedule.views import DeleteEventView

urlpatterns = patterns('',
    url(r'^$', 'timetableapp.views.home', name='home'),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^accounts/login/$', 'timetableapp.views.login'),
    url(r'^accounts/auth/$', 'timetableapp.views.auth_view'),
    url(r'^accounts/logout/$', 'timetableapp.views.logout'),
    url(r'^accounts/loggedin/$', 'timetableapp.views.loggedin'),
    url(r'^accounts/invalid/$', 'timetableapp.views.invalid_login'),
    url(r'^accounts/register/$', 'timetableapp.views.register_user'),
    url(r'^accounts/register_success/$', 'timetableapp.views.register_success'),

    url(r'^timetable/(?P<calendar_slug>[-\w]+)/$',
        'schedule.views.calendar_by_periods',
        name="day_calendar",
        kwargs={'periods': [Day], 'template_name': 'schedule/calendar_day.html'}),

    url(r'^timetable/compact_month/(?P<calendar_slug>[-\w]+)/$',
        'schedule.views.calendar_by_periods',
        name="compact_calendar",
        kwargs={'periods': [Month], 'template_name': 'schedule/calendar_compact_month.html'}),

    url(r'^timetable/month/(?P<calendar_slug>[-\w]+)/$',
        'schedule.views.calendar_by_periods',
        name="month_calendar",
        kwargs={'periods': [Month], 'template_name': 'schedule/calendar_month.html'}),

    url(r'^timetable/week/(?P<calendar_slug>[-\w]+)/$',
        'schedule.views.calendar_by_periods',
        name="week_calendar",
        kwargs={'periods': [Week], 'template_name': 'schedule/calendar_week.html'}),

    url(r'^timetable/daily/(?P<calendar_slug>[-\w]+)/$',
        'schedule.views.calendar_by_periods',
        name="day_calendar",
        kwargs={'periods': [Day], 'template_name': 'schedule/calendar_day.html'}),

    url(r'^lecture/$', 'timetableapp.views.lecture'),
    url(r'^calendar/$', 'schedule.views.calendar'),
) + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

urlpatterns += patterns('',
    url(r'^cal/', include('schedule.urls')),
)


from config import *
from lib import *
from data.newsgroups20 import Newsgroup
from torch.utils.data import DataLoader
from models.CNNText_Inception import CNNText_inception
from tensorboardX import SummaryWriter
from torch import optim
import torch.nn.parallel
import tqdm
import torch
from dixitool.pytorch.module import functional as F
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
#
seed_everything(1995)
# tensorboardX
writer = SummaryWriter()

#gpu
output_device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

print(output_device)
#dataset train,val
#only train now
print("Start loading data")
#variable args comes from config.py
train_ds = Newsgroup(args.data.train_dir,train=True)
test_ds = Newsgroup(args.data.test_dir, train=False)
#dataloader
train_dl = DataLoader(train_ds,batch_size=args.data.dataloader.batch_size,shuffle=True,drop_last=True)
test_dl = DataLoader(test_ds,batch_size=args.data.dataloader.batch_size,shuffle=False)

print("Finish loading data")
net = CNNText_inception(args.model)
net.to(output_device)
net = nn.DataParallel(net, list(range(2)))
criterion = nn.CrossEntropyLoss()

    #optimizer

#loss function CrossEntropyLoss
optimizer = optim.SGD(net.parameters(), lr=args.train.lr , weight_decay=args.train.weight_decay, momentum=args.train.momentum, nesterov=True)

best_acc = 0.0
global_step = 0
for epoch in range(args.train.epochs):
    print("Epoch:{}/{}".format(epoch,args.train.epochs))
    #train
    net.train()
    for idx, data in tqdm.tqdm(enumerate(train_dl)):
        inputs, labels = data
        inputs = inputs.to(output_device)
        labels = labels.to(output_device)
        optimizer.zero_grad()

        outputs = net(inputs)

        loss = criterion(outputs,labels)
        loss.backward()

        optimizer.step()

        writer.add_scalar('train/train_loss',loss.item(),global_step)

        global_step += 1

    #val
    net.eval()
    corrects = 0
    for idx, data in enumerate(test_dl):
        inputs, labels = data
        inputs = inputs.to(output_device)
        labels = labels.to(output_device)
        with torch.no_grad():
            outputs = net(inputs)
            _, preds = torch.max(outputs,dim=1)
            corrects += torch.sum(preds == labels).cpu().item()

    #准确率
    total_acc = float(corrects) / len(test_ds)
    #tensorboardX可视化
    writer.add_scalar('data/val_acc',total_acc,epoch)

    #保存模型参数
    if total_acc>best_acc:
        best_acc = total_acc
        F.save_model('checkpoints', '%.2f' % total_acc, net)
        #保存模型

writer.export_scalars_to_json("./all_scalars.json")
writer.close()


"""
****************************************************************************************************
:copyright (c) 2019-2020 URBANopt, Alliance for Sustainable Energy, LLC, and other contributors.

All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted
provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of conditions
and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice, this list of conditions
and the following disclaimer in the documentation and/or other materials provided with the
distribution.

Neither the name of the copyright holder nor the names of its contributors may be used to endorse
or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
****************************************************************************************************
"""

import os
import shutil
import unittest
from pathlib import Path

from geojson_modelica_translator.geojson_modelica_translator import (
    GeoJsonModelicaTranslator
)
from geojson_modelica_translator.model_connectors.spawn import SpawnConnector
from geojson_modelica_translator.modelica.modelica_runner import ModelicaRunner
from geojson_modelica_translator.system_parameters.system_parameters import (
    SystemParameters
)

class SpawnModelConnectorSingleBuildingTest(unittest.TestCase):
    def setUp(self):
        root_dir, project_name = "tests/model_connectors/output", "spawn_single"

        if os.path.exists(os.path.join(root_dir, project_name)):
            shutil.rmtree(os.path.join(root_dir, project_name))

        # load in the example geojson with a single offie building
        filename = os.path.abspath("tests/model_connectors/data/spawn_geojson_ex1.json")
        self.gj = GeoJsonModelicaTranslator.from_geojson(filename)
        self.gj.scaffold_directory(root_dir, project_name)  # use the GeoJson translator to scaffold out the directory

        # load system parameter data
        filename = os.path.abspath("tests/model_connectors/data/spawn_system_params_ex1.json")
        sys_params = SystemParameters(filename)

        # now test the spawn connector (independent of the larger geojson translator
        self.spawn = SpawnConnector(sys_params)

        for b in self.gj.buildings:
            self.spawn.add_building(b)

    def test_spawn_init(self):
        self.assertIsNotNone(self.spawn)
        self.assertEqual(self.spawn.system_parameters.get_param("buildings.custom")[0]["load_model"], "Spawn", )

    def test_spawn_to_modelica(self):
        self.spawn.to_modelica(self.gj.scaffold)

    def test_spawn_to_modelica_and_run(self):
        self.spawn.to_modelica(self.gj.scaffold)

        # make sure the model can run using the ModelicaRunner class
        mr = ModelicaRunner()
        file_to_run = os.path.abspath(
            os.path.join(self.gj.scaffold.loads_path.files_dir, 'B5a6b99ec37f4de7f94020090', 'coupling.mo'),
        )
        run_path = Path(os.path.abspath(self.gj.scaffold.project_path)).parent
        exitcode = mr.run_in_docker(file_to_run, run_path=run_path)
        self.assertEqual(0, exitcode)

class SpawnModelConnectorTwoBuildingTest(unittest.TestCase):
    def setUp(self):
        root_dir, project_name = "tests/model_connectors/output", "spawn_two_building"

        if os.path.exists(os.path.join(root_dir, project_name)):
            shutil.rmtree(os.path.join(root_dir, project_name))

        # load in the example geojson with a single offie building
        filename = os.path.abspath("tests/model_connectors/data/spawn_geojson_ex2.json")
        self.gj = GeoJsonModelicaTranslator.from_geojson(filename)
        self.gj.scaffold_directory(root_dir, project_name)  # use the GeoJson translator to scaffold out the directory

        # load system parameter data
        filename = os.path.abspath("tests/model_connectors/data/spawn_system_params_ex2.json")
        sys_params = SystemParameters(filename)

        # now test the spawn connector (independent of the larger geojson translator
        self.spawn = SpawnConnector(sys_params)

        for b in self.gj.buildings:
            self.spawn.add_building(b)

    def test_spawn_to_modelica_and_run(self):
        self.spawn.to_modelica(self.gj.scaffold)

        # make sure the model can run using the ModelicaRunner class
        mr = ModelicaRunner()
        file_to_run = os.path.abspath(
            os.path.join(self.gj.scaffold.loads_path.files_dir, 'B5a6b99ec37f4de7f94021950', 'coupling.mo')
        )
        run_path = Path(os.path.abspath(self.gj.scaffold.project_path)).parent
        exitcode = mr.run_in_docker(file_to_run,  run_path=run_path)
        self.assertEqual(0, exitcode)


import math
import numpy as np
import pandas as pd
from scipy.stats import norm
from scipy.interpolate import CubicSpline, interp1d, UnivariateSpline
import matplotlib.pyplot as plt

############################################################
# Simple data input
############################################################
strikes = np.array(range(1000,3000,100))
iv = [0.05, 0.07, 0.08, 0.1, 0.12, 0.13, 0.15, 0.16, 0.18, 0.19,
      0.21, 0.23, 0.24, 0.26, 0.28, 0.29, 0.31, 0.33, 0.34, 0.35]
iv = np.array(list(reversed(iv)))
df_data = pd.DataFrame(list(zip(strikes,iv)), columns=['Strikes', 'IV'])
print('Observed market data of implied volatilities (IV)')
print(df_data)

# helper function to calculate European option price
def option_price(sig, k, is_call):
    ''' Calculate option price under given assumptions.
    Assume zero risk free rate and dividend. F = 2000, T = 1 year
    Args:
        sig (float): the implied volatility corresponding to strike k
        k (float): the strike price
        is_call (boolean): for simplicity, True for call and False for put
    Returns:
        float: the European option price with expiry of 1 year
    '''
    f, T = 2000, 1
    d1 = (math.log(f/k) + 0.5 * sig**2 * T) / (sig * T**0.5)
    d2 = d1 - sig * T**0.5
    if is_call:
        price = f * norm.cdf(d1) - k * norm.cdf(d2)
    else:
        price = k * norm.cdf(-d2) - f * norm.cdf(-d1)
    return price

############################################################
# Look for any butterfly arbitrage
############################################################
call_prices = [option_price(vol, k, is_call=True)
               for vol, k in zip(iv, strikes)]
butterfly_prices = [call_prices[i+1] - 2 * call_prices[i] + call_prices[i-1]
                    for i in range(1, len(call_prices)-1)]
plt.figure()
plt.plot(strikes[1:-1], butterfly_prices, 'o')
plt.vlines(strikes[1:-1], ymin=np.zeros(len(butterfly_prices)), ymax=butterfly_prices, linestyles='dotted')
plt.plot((1000,3000), (0,0), 'r--')
plt.xlabel('strikes serving as butterfly centers with')
plt.ylabel('butterfly prices based on given IV')
plt.xticks(range(1000,3200,200))
plt.title('Prices of butterflies centered at various strikes')
plt.tight_layout()
plt.savefig('butterfly given')
print(pd.DataFrame({'Strikes': strikes[1:-1], 'Butterfly prices': butterfly_prices}))

############################################################
# Interpolate implied volatility with cubic spline
# Compare this with smoothing spline
############################################################
k_range = np.arange(1000,2901,1)

iv_spline_interpolator = CubicSpline(strikes, iv)
cs_interpolated_iv = iv_spline_interpolator(k_range)

iv_smoothing_interpolator = UnivariateSpline(strikes, iv)
smooth_interpolated_iv = iv_smoothing_interpolator(k_range)

plt.figure()
plt.plot(strikes, iv, 'o', label='market data of IV')
plt.plot(k_range, cs_interpolated_iv, label='cubic spline interpolation')
plt.plot(k_range, smooth_interpolated_iv, 'r--', label='smoothing spline interpolation')
plt.xlabel('strike')
plt.ylabel('implied volatility')
plt.xticks(range(1000,3200,200))
plt.title('Interpolate implied volatility (IV) with splines')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('smoothing spline implied vol')

############################################################
# Use cubic-spline-interpolated IV to calculate call price C
# then differentiate C twice for risk neutral pdf
############################################################
call_prices = [option_price(vol, k, is_call=True)
               for vol, k in zip(cs_interpolated_iv, k_range)]
cs_iv_implied_pdf = [call_prices[i+1] - 2 * call_prices[i] + call_prices[i-1]
                     for i in range(1, len(call_prices)-1)]
curve_integral = np.trapz(cs_iv_implied_pdf, x=k_range[1:-1])
label = 'area under curve: {:.2f}'.format(curve_integral)
plt.figure()
plt.plot(k_range[1:-1], cs_iv_implied_pdf, label=label)
plt.xlabel('stock price at expiry')
plt.ylabel('pdf implied by call premium')
plt.xticks(range(1000,3200,200))
plt.title('Implied pdf of S(T) using cubic-spline-interpolated IV')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('pdf with spline-interpolated iv')

############################################################
# Use smoothing-spline-interpolated IV to calculate C
# then differentiate C twice for risk neutral pdf
############################################################
call_prices = [option_price(vol, k, is_call=True)
               for vol, k in zip(smooth_interpolated_iv, k_range)]
smooth_iv_implied_pdf = [call_prices[i+1] - 2 * call_prices[i] + call_prices[i-1]
                         for i in range(1, len(call_prices)-1)]
curve_integral = np.trapz(smooth_iv_implied_pdf, x=k_range[1:-1])
label = 'area under curve: {:.2f}'.format(curve_integral)
plt.figure()
plt.plot(k_range[1:-1], smooth_iv_implied_pdf, 'r--', label=label)
plt.plot((1000,3000), (0,0), 'b:')
plt.xlabel('stock price at expiry')
plt.ylabel('pdf implied by call premium')
plt.xticks(range(1000,3200,200))
plt.title('Implied pdf of S(T) using smoothing-spline-interpolated IV')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('pdf with smooth-interpolated iv')

############################################################
# Use smoothing-spline-interpolated IV to obtain pdf of S(T)
# Estimate the skewness of this pdf
############################################################
mu = np.trapz([f * s for f, s in zip(smooth_iv_implied_pdf, k_range[1:-1])],
              x=k_range[1:-1])
var = -mu**2 + np.trapz([f * s**2 for f, s in zip(smooth_iv_implied_pdf, k_range[1:-1])],
                        x=k_range[1:-1])
std = var**0.5
skewness = np.trapz([f * ((s-mu)/std)**3 for f, s in zip(smooth_iv_implied_pdf, k_range[1:-1])],
                    x=k_range[1:-1])

# use price of OTM digital to show skewness
def digital_price(sig, k, is_call):
    f, T = 2000, 1
    d2 = (math.log(f/k) - 0.5 * sig**2 * T) / (sig * T**0.5)
    if is_call:
        price = norm.cdf(d2)
    else:
        price = norm.cdf(-d2)
    return price

digitals = [digital_price(vol, k, is_call=True) if k > 2000 else digital_price(vol, k, is_call=False)
            for vol, k in zip(smooth_interpolated_iv, k_range)]
plt.plot(k_range, digitals, 'b-')
plt.plot((1000,3000), (0,0), 'k:')
plt.xlabel('strike of OTM digital')
plt.ylabel('price of OTM cash-or-nothing digital')
plt.xticks(range(1000,3200,200))
plt.title('OTM cash-or-nothing digital prices using smoothing-spline-interpolated IV')
plt.tight_layout()
plt.savefig('digital OTM prices')

############################################################
# Interpolate implied volatility with piecewise linear func
############################################################
iv_linear_interpolator = interp1d(strikes, iv, kind='linear')
linear_interpolated_iv = iv_linear_interpolator(k_range)
call_prices = [option_price(vol, k, is_call=True)
               for vol, k in zip(linear_interpolated_iv, k_range)]
linear_implied_pdf = [call_prices[i+1] - 2 * call_prices[i] + call_prices[i-1]
                      for i in range(1, len(call_prices)-1)]
curve_integral = np.trapz(linear_implied_pdf, x=k_range[1:-1])
label = 'area under curve: {:.2f}'.format(curve_integral)
plt.figure()
plt.plot(k_range[1:-1], linear_implied_pdf, label=label)
plt.xlabel('stock price at expiry')
plt.ylabel('pdf implied by call premium')
plt.xticks(range(1000,3200,200))
plt.title('Implied pdf of S(T) using linear-interpolated IV')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('pdf with linear-interpolated iv')

############################################################
# directly use given IV to calculate call prices C
# then twice differentiate C with large step h
# the error in this numerical differentiation is high
# but we include only for completeness
############################################################
call_prices = [option_price(vol, k, is_call=True)
               for vol, k in zip(iv, strikes)]
sparse_pdf = [(call_prices[i+1] - 2 * call_prices[i] + call_prices[i-1]) / ((strikes[i+1] - strikes[i])**2)
              for i in range(1, len(call_prices)-1)]
curve_integral = np.trapz(sparse_pdf, x=strikes[1:-1])
label = 'area under curve: {:.2f}'.format(curve_integral)
plt.figure()
plt.plot(strikes[1:-1], sparse_pdf, label=label)
plt.xlabel('stock price at expiry')
plt.ylabel('pdf implied by call premium')
plt.xticks(range(1000,3200,200))
plt.title('Implied pdf of S(T) using only given IV')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('pdf with only given iv')

############################################################
# use given IV to calculate C
# then interpolate C with cubic spline and take derivative
############################################################
given_calls = [option_price(vol, k, is_call=True)
               for vol, k in zip(iv, strikes)]
call_spline_interpolator = CubicSpline(strikes, given_calls)
call_prices = call_spline_interpolator(k_range)
# plot the cubic spline interpolation of call premiums
plt.figure()
plt.plot(strikes, given_calls, 'o', label='based on given IV data')
plt.plot(k_range, call_prices, label='cubic spline interpolation')
plt.xlabel('strike')
plt.ylabel('calculated call premiums')
plt.xticks(range(1000,3200,200))
plt.title('Interpolate call premiums with cubic spline')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('cubic spline call premium')
# now numerically differentiate with FDM step h=1
cs_call_implied_pdf = [call_prices[i+1] - 2 * call_prices[i] + call_prices[i-1]
                       for i in range(1, len(call_prices)-1)]
curve_integral = np.trapz(cs_call_implied_pdf, x=k_range[1:-1])
label = 'area under curve: {:.2f}'.format(curve_integral)
plt.figure()
plt.plot(k_range[1:-1], cs_call_implied_pdf, label=label)
plt.xlabel('stock price at expiry')
plt.ylabel('pdf implied by spline-interpolated call')
plt.xticks(range(1000,3200,200))
plt.title('Implied pdf of S(T) using interpolated call premiums')
plt.legend(loc='best')
plt.tight_layout()
plt.savefig('pdf with interpolated call premiums')

############################################################
# example 2Y vol curve
############################################################
# 1 year expiry
linear = interp1d([1000,2900], [0.35, 0.05])
iv_1y = linear(k_range)
calls = [option_price(v, k, is_call=True)
         for v, k in zip(iv_1y, k_range)]
pdf_1y = [calls[i+1] - 2 * calls[i] + calls[i-1]
          for i in range(1, len(k_range)-1)]

# 2 year expiry
linear = interp1d([1000, 2900], [0.37, 0.15])
iv_2y = linear(k_range)
calls = [option_price(v, k, is_call=True)
         for v, k in zip(iv_2y, k_range)]
pdf_2y = [calls[i+1] - 2 * calls[i] + calls[i-1]
          for i in range(1, len(k_range)-1)]

fig, ax1 = plt.subplots()
ax1.plot(k_range[1:-1], pdf_1y, 'r--', label='1Y density')
ax1.plot(k_range[1:-1], pdf_2y, 'b-', label='2Y density')
ax1.set_xlabel('strike')
ax1.set_ylabel('probability density function')
ax2 = ax1.twinx()
ax2.plot(k_range, iv_1y, label='1Y implied vol')
ax2.plot(k_range, iv_2y, label='2Y implied vol')
ax2.set_ylabel('implied vol')
ax1.legend(loc=1)
ax2.legend(loc=2)
fig.tight_layout()
fig.savefig('example 2y density')


from typing import List, no_type_check, Any

from fugue.collections.partition import PartitionCursor
from fugue.dataframe import (
    ArrayDataFrame,
    DataFrame,
    DataFrames,
    LocalDataFrame,
    to_local_bounded_df,
)
from fugue.exceptions import FugueWorkflowError
from fugue.execution import SQLEngine
from fugue.execution.execution_engine import _generate_comap_empty_dfs
from fugue.extensions.processor import Processor
from fugue.extensions.transformer import CoTransformer, Transformer, _to_transformer
from triad.collections import ParamDict
from triad.collections.schema import Schema
from triad.utils.assertion import assert_or_throw
from triad.utils.convert import to_instance, to_type

class RunTransformer(Processor):
    @no_type_check
    def process(self, dfs: DataFrames) -> DataFrame:
        df = dfs[0]
        tf = _to_transformer(
            self.params.get_or_none("transformer", object),
            self.params.get_or_none("schema", object),
        )
        tf._workflow_conf = self.execution_engine.conf
        tf._params = self.params.get("params", ParamDict())  # type: ignore
        tf._partition_spec = self.partition_spec  # type: ignore
        ie = self.params.get("ignore_errors", [])
        self._ignore_errors = [to_type(x, Exception) for x in ie]

        if isinstance(tf, Transformer):
            return self.transform(df, tf)
        else:
            return self.cotransform(df, tf)

    def transform(self, df: DataFrame, tf: Transformer) -> DataFrame:
        tf._key_schema = self.partition_spec.get_key_schema(df.schema)  # type: ignore
        tf._output_schema = Schema(tf.get_output_schema(df))  # type: ignore
        tr = _TransformerRunner(df, tf, self._ignore_errors)  # type: ignore
        return self.execution_engine.map(
            df=df,
            map_func=tr.run,
            output_schema=tf.output_schema,  # type: ignore
            partition_spec=tf.partition_spec,
            on_init=tr.on_init,
        )

    @no_type_check
    def cotransform(self, df: DataFrame, tf: CoTransformer) -> DataFrame:
        assert_or_throw(
            df.metadata.get("serialized", False), "must use serialized dataframe"
        )
        tf._key_schema = df.schema - list(df.metadata["serialized_cols"].values())
        # TODO: currently, get_output_schema only gets empty dataframes
        empty_dfs = _generate_comap_empty_dfs(
            df.metadata["schemas"], df.metadata.get("serialized_has_name", False)
        )
        tf._output_schema = Schema(tf.get_output_schema(empty_dfs))
        tr = _CoTransformerRunner(df, tf, self._ignore_errors)
        return self.execution_engine.comap(
            df=df,
            map_func=tr.run,
            output_schema=tf.output_schema,
            partition_spec=tf.partition_spec,
            on_init=tr.on_init,
        )

class RunJoin(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        if len(dfs) == 1:
            return dfs[0]
        how = self.params.get_or_throw("how", str)
        on = self.params.get("on", [])
        df = dfs[0]
        for i in range(1, len(dfs)):
            df = self.execution_engine.join(df, dfs[i], how=how, on=on)
        return df

class RunSetOperation(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        if len(dfs) == 1:
            return dfs[0]
        how = self.params.get_or_throw("how", str)
        func: Any = {
            "union": self.execution_engine.union,
            "subtract": self.execution_engine.subtract,
            "intersect": self.execution_engine.intersect,
        }[how]
        distinct = self.params.get("distinct", True)
        df = dfs[0]
        for i in range(1, len(dfs)):
            df = func(df, dfs[i], distinct=distinct)
        return df

class Distinct(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        assert_or_throw(len(dfs) == 1, FugueWorkflowError("not single input"))
        return self.execution_engine.distinct(dfs[0])

class RunSQLSelect(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        statement = self.params.get_or_throw("statement", str)
        engine = self.params.get_or_none("sql_engine", object)
        if engine is None:
            engine = self.execution_engine.default_sql_engine
        elif not isinstance(engine, SQLEngine):
            engine = to_instance(engine, SQLEngine, args=[self.execution_engine])
        return engine.select(dfs, statement)

class Zip(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        how = self.params.get("how", "inner")
        partition_spec = self.partition_spec
        # TODO: this should also search on workflow conf
        temp_path = self.params.get_or_none("temp_path", str)
        to_file_threshold = self.params.get_or_none("to_file_threshold", object)
        return self.execution_engine.zip_all(
            dfs,
            how=how,
            partition_spec=partition_spec,
            temp_path=temp_path,
            to_file_threshold=to_file_threshold,
        )

class Rename(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        assert_or_throw(len(dfs) == 1, FugueWorkflowError("not single input"))
        columns = self.params.get_or_throw("columns", dict)
        return dfs[0].rename(columns)

class DropColumns(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        assert_or_throw(len(dfs) == 1, FugueWorkflowError("not single input"))
        if_exists = self.params.get("if_exists", False)
        columns = self.params.get_or_throw("columns", list)
        if if_exists:
            columns = set(columns).intersection(dfs[0].schema.keys())
        return dfs[0].drop(list(columns))

class SelectColumns(Processor):
    def process(self, dfs: DataFrames) -> DataFrame:
        assert_or_throw(len(dfs) == 1, FugueWorkflowError("not single input"))
        columns = self.params.get_or_throw("columns", list)
        return dfs[0][columns]

class _TransformerRunner(object):
    def __init__(
        self, df: DataFrame, transformer: Transformer, ignore_errors: List[type]
    ):
        self.schema = df.schema
        self.metadata = df.metadata
        self.transformer = transformer
        self.ignore_errors = tuple(ignore_errors)

    def run(self, cursor: PartitionCursor, df: LocalDataFrame) -> LocalDataFrame:
        self.transformer._cursor = cursor  # type: ignore
        df._metadata = self.metadata
        if len(self.ignore_errors) == 0:
            return self.transformer.transform(df)
        else:
            try:
                return to_local_bounded_df(self.transformer.transform(df))
            except self.ignore_errors:  # type: ignore
                return ArrayDataFrame([], self.transformer.output_schema)

    def on_init(self, partition_no: int, df: DataFrame) -> None:
        s = self.transformer.partition_spec
        self.transformer._cursor = s.get_cursor(  # type: ignore
            self.schema, partition_no
        )
        df._metadata = self.metadata
        self.transformer.on_init(df)

class _CoTransformerRunner(object):
    def __init__(
        self, df: DataFrame, transformer: CoTransformer, ignore_errors: List[type]
    ):
        self.schema = df.schema
        self.metadata = df.metadata
        self.transformer = transformer
        self.ignore_errors = tuple(ignore_errors)

    def run(self, cursor: PartitionCursor, dfs: DataFrames) -> LocalDataFrame:
        self.transformer._cursor = cursor  # type: ignore
        if len(self.ignore_errors) == 0:
            return self.transformer.transform(dfs)

        else:
            try:
                return to_local_bounded_df(self.transformer.transform(dfs))
            except self.ignore_errors:  # type: ignore
                return ArrayDataFrame([], self.transformer.output_schema)

    def on_init(self, partition_no: int, dfs: DataFrames) -> None:
        s = self.transformer.partition_spec
        self.transformer._cursor = s.get_cursor(  # type: ignore
            self.schema, partition_no
        )
        self.transformer.on_init(dfs)


from skimage import feature, img_as_ubyte
import os
import pandas as pd
import numpy as np
import cv2

path = '.\\Data Test\\Malignant\\1.bmp'
# path = '.\\Data Test1\\Benign\\b1.png'
class glcm4s(object):
    data = []

    def __init__(self, pict):
        self.pict = pict
        self.data = []

    def glcm(self):
        hasil = {
            "energy": 0,
            "homogeneity": 0,
            "contrast": 0,
            "correlation": 0
        }
        degrees = [0,45,90,135]
        img_read = cv2.imread(self.pict, 0)
        img_conv = img_as_ubyte(img_read)
        for degree in degrees:
            glcm = feature.greycomatrix(img_conv, [1], [degree], symmetric=True, normed=True)
            for prop in hasil.keys():
                self.data.append(float(feature.greycoprops(glcm, prop)))

# pict = []

# for jenis in os.listdir(path):
#     folder = path+jenis
#     for isi in os.listdir(folder):
#         data = []
#         gambar = path+jenis+'\\'+isi
#         img = glcm4s(gambar)
#         img.glcm()
#         if jenis == "Benign":
#             data.append("Benign")
#         elif jenis == "Malignant":
#             data.append("Malignant")
#         pict.append(data)

# print(img.data)
# pd.DataFrame(pict).to_csv('./file.csv', index=False, header=False)

img = glcm4s(path)
img.glcm()
print(img.data)


# Copyright 2019-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Example parallel workflow
"""
import json
import logging
from datetime import datetime
from airflow import DAG
from airflow import AirflowException
from airflow.operators.python_operator import PythonOperator
from airflow.sensors.cord_workflow_plugin import CORDEventSensor, CORDModelSensor
from airflow.operators.cord_workflow_plugin import CORDModelOperator

log = logging.getLogger(__name__)
args = {
    # hard coded date
    'start_date': datetime(2019, 1, 1),
    'owner': 'iychoi'
}

dag_parallel_cord = DAG(
    dag_id='parallel_cord_workflow',
    default_args=args,
    # this dag will be triggered by external systems
    schedule_interval=None
)
dag_parallel_cord.doc_md = __doc__

dag_parallel_cord_admin = DAG(
    dag_id='parallel_cord_workflow_admin',
    default_args=args,
    # this dag will be triggered by external systems
    schedule_interval=None
)
dag_parallel_cord_admin.doc_md = __doc__

def on_onu_event(model_accessor, message, **kwargs):
    log.info('onu.events: received an event - %s' % message)

def on_auth_event(model_accessor, message, **kwargs):
    log.info('authentication.events: received an event - %s' % message)

def on_dhcp_event(model_accessor, message, **kwargs):
    log.info('dhcp.events: received an event - %s' % message)

def on_model_event(model_accessor, message, **kwargs):
    log.info('model event: received an event - %s' % message)

onu_event_sensor = CORDEventSensor(
    task_id='onu_event_sensor',
    topic='onu.events',
    key_field='serialNumber',
    controller_conn_id='local_cord_controller',
    poke_interval=5,
    dag=dag_parallel_cord
)

onu_event_handler = CORDModelOperator(
    task_id='onu_event_handler',
    python_callable=on_onu_event,
    cord_event_sensor_task_id='onu_event_sensor',
    dag=dag_parallel_cord
)

auth_event_sensor = CORDEventSensor(
    task_id='auth_event_sensor',
    topic='authentication.events',
    key_field='serialNumber',
    controller_conn_id='local_cord_controller',
    poke_interval=5,
    dag=dag_parallel_cord
)

auth_event_handler = CORDModelOperator(
    task_id='auth_event_handler',
    python_callable=on_auth_event,
    cord_event_sensor_task_id='auth_event_sensor',
    dag=dag_parallel_cord
)

dhcp_event_sensor = CORDEventSensor(
    task_id='dhcp_event_sensor',
    topic='dhcp.events',
    key_field='serialNumber',
    controller_conn_id='local_cord_controller',
    poke_interval=5,
    dag=dag_parallel_cord
)

dhcp_event_handler = CORDModelOperator(
    task_id='dhcp_event_handler',
    python_callable=on_dhcp_event,
    cord_event_sensor_task_id='dhcp_event_sensor',
    dag=dag_parallel_cord
)

att_model_event_sensor = CORDModelSensor(
    task_id='att_model_event_sensor',
    model_name='AttWorkflowDriverServiceInstance',
    key_field='serialNumber',
    controller_conn_id='local_cord_controller',
    poke_interval=5,
    dag=dag_parallel_cord_admin
)

att_model_event_handler = CORDModelOperator(
    task_id='att_model_event_handler',
    python_callable=on_model_event,
    cord_event_sensor_task_id='att_model_event_sensor',
    dag=dag_parallel_cord_admin
)

# handle standard flow
onu_event_sensor >> onu_event_handler >> auth_event_sensor >> auth_event_handler >> dhcp_event_sensor >> dhcp_event_handler
# handle admin flow
att_model_event_sensor >> att_model_event_handler


"""5 Train Contrastive.

Train or fine-tune InceptionV3 architecture to fit van-Gogh data set.

Author: Lucas David -- <lucasolivdavid@gmail.com>
Licence: MIT License 2016 (c)

"""

import os

import tensorflow as tf
from keras import backend as K, callbacks, optimizers
from sacred import Experiment
from sacred.utils import apply_backspaces_and_linefeeds

from connoisseur.models import build_siamese_model
from connoisseur.utils.image import BalancedDirectoryPairsSequence, ImageDataGenerator

from base import contrastive_loss

ex = Experiment('5-train-contrastive')

ex.captured_out_filter = apply_backspaces_and_linefeeds

@ex.config
def config():
    image_shape = [299, 299, 3]
    batch_size = 150
    device = '/gpu:0'
    data_dir = '/datasets/vangogh'
    train_dir = '/datasets/vangogh/vgdb_2016/extracted_patches_256/train'
    valid_dir = '/datasets/vangogh/vgdb_2016/extracted_patches_256/valid'
    balanced_classes = True

    train_augmentations = []
    train_shuffle = True
    train_dataset_seed = 14
    valid_augmentations = []
    valid_shuffle = True
    valid_dataset_seed = 9

    arch = 'inejc'
    network_weights = None
    pre_trained_weights = None

    opt_params = {'lr': 0.005}
    ckpt_file = '/work/vangogh/5-train-contrastive/ckpt/optimal.hdf5'
    nb_epoch = 10000
    train_samples_per_epoch = 13350
    nb_val_samples = 2048
    nb_worker = 1
    early_stop_patience = 300
    reduce_lr_on_plateau_patience = 100
    tensorboard_file = './logs/loss:contrastive'

    initial_epoch = 0

@ex.automain
def run(batch_size, data_dir, train_dir, valid_dir, balanced_classes,
        image_shape,
        train_augmentations, train_shuffle, train_dataset_seed,
        valid_augmentations, valid_shuffle, valid_dataset_seed,
        device, opt_params, ckpt_file, pre_trained_weights,
        arch, network_weights,
        train_samples_per_epoch, nb_epoch, nb_val_samples, nb_worker,
        early_stop_patience, reduce_lr_on_plateau_patience, tensorboard_file,
        initial_epoch):
    os.makedirs(os.path.dirname(ckpt_file), exist_ok=True)

    tf_config = tf.ConfigProto(allow_soft_placement=True)
    tf_config.gpu_options.allow_growth = True
    sess = tf.Session(config=tf_config)
    K.set_session(sess)

    g = ImageDataGenerator(
        featurewise_center=True,
        horizontal_flip=True,
        vertical_flip=True,
        rotation_range=30,
        height_shift_range=.2,
        width_shift_range=.2,
        rescale=2. / 255.,
        fill_mode='reflect')
    g.mean = 1.
    train_data = BalancedDirectoryPairsSequence(
        train_dir,
        image_data_generator=g, target_size=image_shape[:2],
        augmentations=train_augmentations, batch_size=batch_size,
        balanced_classes=balanced_classes, shuffle=train_shuffle,
        seed=train_dataset_seed)
    valid_data = BalancedDirectoryPairsSequence(
        valid_dir,
        image_data_generator=g, target_size=image_shape[:2],
        augmentations=valid_augmentations, batch_size=batch_size,
        balanced_classes=balanced_classes, shuffle=valid_shuffle,
        seed=valid_dataset_seed)

    with tf.device(device):
        model = build_siamese_model(x_shape=image_shape, arch=arch, weights=network_weights)
        opt = optimizers.Adam(**opt_params)
        model.compile(optimizer=opt, loss=contrastive_loss)

        if pre_trained_weights:
            model.load_weights(pre_trained_weights)
        elif initial_epoch > 0:
            print('re-loading weights...')
            model.load_weights(ckpt_file)

        print('training network...')
        try:
            model.fit_generator(
                train_data, samples_per_epoch=train_samples_per_epoch,
                nb_epoch=nb_epoch,
                validation_data=valid_data, nb_val_samples=nb_val_samples,
                nb_worker=nb_worker,
                callbacks=[
                    callbacks.ReduceLROnPlateau(min_lr=1e-10, patience=reduce_lr_on_plateau_patience),
                    callbacks.EarlyStopping(patience=early_stop_patience),
                    callbacks.TensorBoard(tensorboard_file, histogram_freq=5),
                    callbacks.ModelCheckpoint(ckpt_file, verbose=1, save_best_only=True),
                ],
                initial_epoch=initial_epoch)
        except KeyboardInterrupt:
            print('training interrupted by user.')
        else:
            print('done.')


import sys
import numpy as np
import pyqtgraph as pg
from PySide import QtCore, QtGui
from ventana_MP_conversion import Ui_Form
import matplotlib as mpl

mpl.use('Qt4Agg')
mpl.rcParams['backend.qt4']='PySide'
import matplotlib.pyplot as plt
from show_cursor import SnaptoCursor, Cursor

class Reactor_mp_conversion(QtGui.QWidget, Ui_Form):

    def __init__(self):

        QtGui.QWidget.__init__(self)
        self.setupUi(self)
        self.le_molA.setText('1')
        self.le_molB.setText('1')
        self.cb_orden.setCurrentIndex(2)
        self.le_temp_fin.setText('393')
        self.le_concentracion_ini.setText('6.65')
        self.le_ener_act.setText('83680')
        self.le_k.setText('55000000')
        self.le_volumen_2.setText('912')
        self.le_caudal.setText('1.26')

        self.deltaT = 0.5
        self.R = 8.3143
        self.plot.setLabel('left', 'Conversión')
        self.plot.setLabel('bottom', 'Volumen', units='L')

        self.btn_mostrar_resultados.setDisabled(True)

        self.btn_ejecutar.clicked.connect(self.ejecutar)
        self.btn_mostrar_resultados.clicked.connect(self.mostrar_resultado)
        self.btn_reset.clicked.connect(self.resetear)
        self.btn_cerrar.clicked.connect(self.close)

    def leerDatos(self):

        self.molA = float(self.le_molA.text())
        self.molB = float(self.le_molB.text())
        self.orden = int(self.cb_orden.currentText()[0].strip())
        self.temp_fin = float(self.le_temp_fin.text())
        self.cao = float(self.le_concentracion_ini.text())
        self.ener_act = float(self.le_ener_act.text())
        self.k = float(self.le_k.text())
        self.volumen = float(self.le_volumen_2.text())
        self.caudal = float(self.le_caudal.text())

    def calcularK(self):

        self.K = self.k * np.exp(-self.ener_act / (self.R * self.temp_fin))

    def f0(self, vol):
        return vol * self.K / self.caudal / self.cao

    def f1(self, vol):
        return vol * self.K / (self.caudal + self.molA * vol * self.K)

    def f2(self, vol):

        a = 1
        b = -((2 / self.molA) + (self.caudal / vol / self.K / self.cao))
        c = 1 / (self.molA ** 2)

        xf1 = (-b + ((b ** 2 - 4 * a * c)) ** 0.5) / (2 * a)
        xf2 = (-b - ((b ** 2 - 4 * a * c)) ** 0.5) / (2 * a)

        if xf1 <= 1:
            return xf1
        else:
            return xf2

    def plotear(self):

        self.x = np.arange(0.5, self.volumen+self.deltaT, self.deltaT)

        if self.orden == 0:
            f = np.vectorize(self.f0)

        elif self.orden == 1:
            f = np.vectorize(self.f1)

        else:
            f = np.vectorize(self.f2)

        self.y = f(self.x)

        self.plot.clear()
        curve = self.plot.plot(self.x, self.y, pen=None)  ## setting pen=None disables line drawing
        curve.curve.setClickable(True)
        curve.setPen('w')  ## white pen

        curve.setShadowPen(pg.mkPen((70, 70, 30), width=6, cosmetic=True))

    def resetear(self):

        self.le_molA.clear()
        self.le_molB.clear()
        self.cb_orden.setCurrentIndex(0)
        self.le_temp_fin.clear()
        self.le_concentracion_ini.clear()
        self.le_ener_act.clear()
        self.le_k.clear()
        self.le_caudal.clear()
        self.le_volumen.clear()
        self.le_volumen_2.clear()
        self.le_xa.clear()
        self.plot.clear()

        self.btn_mostrar_resultados.setDisabled(True)

    def ejecutar(self):

        if self.le_molA.text() == "" or self.le_molB.text() == "" or self.le_temp_fin.text() == "" or self.le_concentracion_ini.text() == "" or self.le_ener_act.text() == "" or self.le_k.text() == "" or self.le_caudal.text() == "" or self.le_volumen_2.text() == "":
            QtGui.QMessageBox.warning(self, "Error en la ejecución", "Alguna/s de las celdas de datos está/n vacía/s. Por favor, revise que ha introducido correctamente todos los datos.", QtGui.QMessageBox.Cancel, QtGui.QMessageBox.NoButton)
        else:
            self.leerDatos()
            self.calcularK()
            self.plotear()

            self.le_volumen.setText(str(self.volumen))

            if self.orden == 0:
                self.le_xa.setText(str(float("{0:.2f}".format(self.f0(self.volumen)))))

            elif self.orden == 1:
                self.le_xa.setText(str(float("{0:.2f}".format(self.f1(self.volumen)))))

            else:
                self.le_xa.setText(str(float("{0:.2f}".format(self.f2(self.volumen)))))

            self.btn_mostrar_resultados.setEnabled(True)

    def mostrar_resultado(self):

        self.fig, self.conv = plt.subplots()

        self.conv.set_title('RESULTADO')

        self.conv.set_xlim([min(self.x), max(self.x)])
        self.conv.set_ylim([min(self.y), max(self.y)])

        self.conv.set_xlabel('Volumen (L)')
        self.conv.set_ylabel('Conversión')

        # cursor = Cursor(ax)
        self.cursor = SnaptoCursor(self.conv, self.x, self.y)
        plt.connect('motion_notify_event', self.cursor.mouse_move)

        self.conv.plot(self.x, self.y, linewidth = 2)

        # Hace que matlplolib controle todas las figuras con sus propios hilos de forma independiente a la gui principal
        # sustituye a la funcion plt.show()
        plt.ion()
        plt.show()

if __name__ == '__main__':

    try:
        app = QtGui.QApplication(sys.argv)
    except RuntimeError:
        pass

    # MainWindow = QtGui.QWidget()
    ui = Reactor_mp_conversion()
    # ui.setupUi(MainWindow)
    ui.show()
    app.exec_()


#! /usr/bin/python3

from __future__ import print_function, division

import csv
from humanfriendly.terminal import output, ansi_wrap, warning
import numpy as np
import numpy as np
import pandas as pd
from scipy import stats
from scipy.optimize import leastsq
from prettytable import PrettyTable
from itertools import product
from math import floor, ceil
from utils import *

MISS=-1

class NumSegsAnalyzer(DataAnalyzer):
    def __init__(self, history, final_num=35, seg_count=7):
        super(NumSegsAnalyzer, self).__init__(history)
        self.final_num = final_num
        self.change_seg_count(seg_count)

    def change_seg_count(self, seg_count):
        assert(seg_count > 0 and seg_count <= self.final_num)
        self.seg_count = seg_count
        self.seg_size = ceil(self.final_num/self.seg_count)
        self._segs_hit_miss_table = []

    def seg_index(self, num):
        assert(num >= 1 and num <= self.final_num)
        idx = floor((num-1)/self.seg_size)
        assert(idx < self.seg_count)
        return idx

    def seg_numbers(self, idx):
        assert(idx < self.seg_count)
        end = (idx + 1) * self.seg_size + 1
        start = end - self.seg_size
        return list(range(start, end))

    def seg_hit_miss(self, idx):
        return [row[idx] for row in self.segs_hit_miss_table]

    @property
    def segs_hit_miss_table(self):
        self._segs_hit_miss_table = []
        for row in self._data:
            row_hit_miss_table = self.row_hit_miss(row)
            self._segs_hit_miss_table.append(row_hit_miss_table)
        return self._segs_hit_miss_table

    def row_hit_miss(self, row):
        ret = [0] * self.seg_count
        for num in row:
            seg_idx = self.seg_index(num)
            ret[seg_idx] += 1

        for i in range(0, len(ret)):
            if ret[i] == 0:
                ret[i] = MISS
        return ret

    def analyze_last(self, count=None):
        self.last_n(count)
        self.analyze()

    def analyze_range(self, start, end):
        self.set_range(start, end)
        self.analyze()

    def analyze(self):
        # his is the data need to analyze
        prt_div()
        prt_title("history segments analyzing, records {}".format(self._data_count))
        prt_title("star and end {}".format(str(self._data_range)))

        for item in self.segs_hit_miss_table:
            print(item)

        for idx in range(0, self.seg_count):
            temp =  self.seg_hit_miss(idx)
            print(temp, self.seg_numbers(idx))

class NumSegsFilter(object):
    def __init__(self, analyzer):
        self.analyzer = analyzer

    @filter_deco
    def miss_seg_rule(self, guesses, min_miss = 1, max_miss = 3):
        warning("Must miss some segments from {} to {}".format(min_miss, max_miss))
        ret = []
        for row in guesses:
            h_m_table = self.analyzer.row_hit_miss(row)
            miss_count = h_m_table.count(MISS)
            if not between(miss_count, min_miss, max_miss):
                continue
            ret.append(row)
        return ret

    @filter_deco
    def no_miss_seg_rule(self, guesses):
        warning("Must hit all segments")
        ret = []
        for row in guesses:
            h_m_table = self.analyzer.row_hit_miss(row)
            miss_count = h_m_table.count(MISS)
            if miss_count > 0:
                continue
            ret.append(row)
        return ret

    def _same_seg_miss(self, previous, current):
        h_m_table = self.analyzer.row_hit_miss(current)
        for idx in range(0, len(previous)):
            if previous[idx] <= MISS and h_m_table[idx] <= MISS:
                return True
        return False

    @filter_deco
    def no_continous_miss_seg(self, guesses):
        warning("Must has the segment if previous miss it!")
        ret = []
        previous = self.analyzer.segs_hit_miss_table[-1]
        for row in guesses:
            if self._same_seg_miss(previous, row):
                continue
            ret.append(row)
        return ret

    @filter_deco
    def no_pattern_as_previous(self, guesses):
        warning("no seg pattern same as previouse one!")
        ret = []
        previous = self.analyzer.segs_hit_miss_table[-1]
        for row in guesses:
            row_pattern = self.analyzer.row_hit_miss(row)
            for n1, n2 in zip(previous, row_pattern):
                if n1 * n2 < 0: # differnt sign
                    ret.append(row)
                    break
        return ret

    @cls_deco
    def filter(self, guesses):
        if self.analyzer.seg_count <= 3:
            guesses = self.no_miss_seg_rule(guesses)
        elif self.analyzer.seg_count <= 5:
            guesses = self.no_continous_miss_seg(guesses)
        elif self.analyzer.seg_count <=7:
            guesses = self.miss_seg_rule(guesses, 0, 3)
            guesses = self.no_pattern_as_previous(guesses)
        return guesses


# -*- coding: utf-8 -*-
"""
Created on Sat Jun 10 10:21:35 2017

@author: new_xuyangcao
"""

import cv2
import matplotlib.pyplot as plt

if __name__ == '__main__':

    # load and transform image
    roi_with_bg = cv2.imread('roi_with_bg.bmp')
    roi_without_bg = cv2.imread('roi_without_bg.bmp')
    gray_without_bg = cv2.cvtColor(roi_without_bg, cv2.COLOR_RGB2GRAY)
    roi_binary = cv2.imread('roi_binary.bmp')
    roi_binary = cv2.cvtColor(roi_binary, cv2.COLOR_BGR2GRAY)

    #histogram of the roi
    hist = cv2.calcHist([roi_without_bg], [0], None, [256], [0, 256])
    hist[0] = 0
    plt.figure,  plt.plot(hist, color = 'g'), plt.title('histogram of roi')

    # find and draw outer edges
    contours, _ = cv2.findContours(roi_binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    cv2.drawContours(roi_with_bg, contours, -1, (255, 0, 0), 2)

    for th in xrange(50, 120, 10):
        #threshold, find, and draw contours
        _, bw = cv2.threshold(gray_without_bg, th, 1, cv2.THRESH_BINARY)
        bw1 = bw.copy()
        contours, _ = cv2.findContours(bw, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
        img1 = roi_with_bg.copy()
        cv2.drawContours(img1, contours, -1, (0, 255, 255), 1)
        # show
        plt.figure(), plt.imshow(img1, cmap = 'gray'), plt.title(str(th)), plt.xticks([]), plt.yticks([])
        #plt.figure(), plt.imshow(bw1, cmap = 'gray'), plt.title(str(th)), plt.xticks([]), plt.yticks([])


#!/usr/bin/python
# -*- coding: utf-8 -*-
import pyforms
import genetic_algorithm
import function_parser as fp
from pyforms.basewidget import BaseWidget
from pyforms.controls import ControlText
from pyforms.controls import ControlButton
from pyforms.controls import ControlTextArea
from pyforms.controls import ControlCombo
from pyforms.controls import ControlMatplotlib
from pyforms.controls import ControlNumber
import os

__author__ = "Ricardo Ribeiro"
__credits__ = ["Ricardo Ribeiro"]
__license__ = "MIT"
__version__ = "0.0"
__maintainer__ = "Ricardo Ribeiro"
__email__ = "ricardojvr@gmail.com"
__status__ = "Development"

class GeneticAlgorithms(BaseWidget):

    def __init__(self):
        super(GeneticAlgorithms, self).__init__('GeneticAlgorithms')

        # Definition of the forms fields
        self._crossover_prob = ControlNumber('Choose crossover probability', minimum=0, maximum=1, decimals=3, step=0.10, default=0.1)
        self._mutation_prob = ControlNumber("Choose mutation probability", minimum=0, maximum=1, decimals=3, step=0.10, default=0.6)
        self._num_individuals = ControlNumber('Choose number of individuals', minimum=2, maximum=100, step=1, default=10)
        self._num_generations = ControlNumber('Choose number of generations', minimum=1, maximum=100, step=1, default=10)
        self._function = ControlText('Function f(x,y)=')
        self._x_start = ControlNumber('X start', decimals=3, minimum=-1000, maximum=1000, step=0.1, default=-1)
        self._x_end = ControlNumber('X end', decimals=3, minimum=-1000, maximum=1000, step=0.1, default=1)
        self._y_start = ControlNumber('Y start', decimals=3, minimum=-1000, maximum=1000, step=0.1, default=-1)
        self._y_end = ControlNumber('Y end', decimals=3, step=0.1, default=1)
        self._loaddatabutton = ControlButton('Load data')
        self._showfunctionbutton = ControlButton('Show function')
        self._showfunctionimage = ControlMatplotlib()
        self._terminal = ControlTextArea()
        self._terminal.readonly
        # Define the button action
        self._loaddatabutton.value = self.__buttonAction
        self._showfunctionbutton.value = self.__function_plotAction
        # Define the organization of the forms
        self._formset = [{
            'Create': [('_function', '_x_start', '_x_end', '_y_start', '_y_end'),
                       ('_crossover_prob', '_mutation_prob', '_num_individuals', '_num_generations'),
                       ('_loaddatabutton'),
                       ('_terminal')
                       ],
            'Show function': [('_showfunctionbutton'),
                              ('_showfunctionimage')],
            'Solve': []
        }]

    def __function_plotAction(self):
        try:
            function_plot = fplot.show_function_plot(function_to_print=self._function.value,
                                     x_domain=[self._x_start.value, self._x_end.value],
                                     y_domain=[self._y_start.value, self._y_end.value])
            self._showfunctionimage.value = 10
        except Exception as e:
            print(e)

    def __buttonAction(self):
        """Button action event"""
        try:
            # initalization of genetic algorithm
            g = genetic_algorithm.GA(
                gui_x_domain=[self._x_start.value, self._x_end.value],
                gui_y_domain=[self._y_start.value, self._y_end.value],
                gui_crossover_prob=self._crossover_prob.value,
                gui_mutation_prob=self._mutation_prob.value,
                gui_num_ind=int(self._num_individuals.value),
                gui_num_gen=int(self._num_generations.value)
            )
            new = ''
            for generation_index in range(g.num_generations):
                new = new + 'Generation ' + str(generation_index+1) + ';\n' + ' Population: '
                new = new + str(g.pop_float) + ';\n' + ' Function value: '
                fitness = fp.fitness_function(self._function.value, g.pop_float)
                new = new + str(fitness) + '\n'
                parents = g.select_parents(fitness)
                g.pop = g.crossover(parents)
                g.pop = g.mutation()
                g.pop = g.bin2int()
                g.pop_float = g.relaxation_function()

                # for generation_index, generation in enumerate(range(g.num_generations)):
            self._terminal.value = new
            self._terminal.finishEditing()
        except SyntaxError:
            self._terminal.value = 'Podaj funkcję'
        except Exception as e:
            self._terminal.value = e

##################################################################################################################
##################################################################################################################
##################################################################################################################

# Execute the application
if __name__ == "__main__":     pyforms.start_app(GeneticAlgorithms)

# Use dictionaries for tabs
        # Use the sign '=' for a vertical splitter
        # Use the signs '||' for a horizontal splitter

        # 'Create:'['_file','_activationfunction','_learningfactor'],
        # 'Solve:'['_fullname']
        # self.mainmenu = [
        #     {'File': [
        #         {'Open': self.__openEvent},
        #         '-',
        #         {'Save': self.__saveEvent},
        #         {'Save as': self.__saveAsEvent}
        #     ]
        #     }
        # ]


from django.test import TestCase
from projects.models import Project

class ProjectTest(TestCase):
    """
    Here all tests are defined that
    are run against the Project models
    """

    def test_str(self):
        project = Project(name='Test Project')
        self.assertEqual(str(project), 'Test Project')

    def test_add_project(self):
        project = Project(name='Test', category='Test Cat',
                          description='Test Desc')
        project.save()
        self.assertEqual(project.name, "Test")
        self.assertEqual(project.category, "Test Cat")
        self.assertEqual(project.description, "Test Desc")


from __future__ import annotations

from contextlib import contextmanager
from datetime import datetime as dt
from pathlib import Path
import sqlite3
from typing import Any, Dict, Generator, List, Optional, Tuple, Union
from warnings import warn

import numpy as np
import pandas as pd

from .api import ButlerOverwriteWarning, MatrixEntry

class MatrixButler(object):

    _MATRIX_EXTENSION = '.bin'
    _SUBDIRECTORY_NAME = 'emmebin'
    _DB_NAME = "matrix_directory.sqlite"

    def __init__(self, *args):
        butler_path, db, zone_system, fortran_max_zones = args
        self._path: Path = butler_path
        self._connection: sqlite3.Connection = db
        self._zone_system: pd.Index = zone_system
        self._max_zones_fortran: int = fortran_max_zones

        self._committing: bool = True

    @staticmethod
    def create(parent_directory: Union[str, Path], zone_system: Union[pd.Index, List[int]],
               fortran_max_zones: int) -> MatrixButler:
        """Creates a new (or clears and initializes and existing) MatrixButler.

        Args:
            parent_directory (Union[str, Path]): The parent directory in which to keep the Butler.
            zone_system (Union[pd.Index, List[int]]): The zone system to conform to.
            fortran_max_zones (int): The total number of zones expected by the FORTRAN matrix reader.

        Returns:
            MatrixButler instance.
        """
        parent_directory = Path(parent_directory)
        zone_system = pd.Index(zone_system, dtype=np.int64)
        fortran_max_zones = int(fortran_max_zones)

        butler_path = parent_directory / MatrixButler._SUBDIRECTORY_NAME
        butler_path.mkdir(parents=True, exist_ok=True)

        dbfile = butler_path / MatrixButler._DB_NAME
        db_exists = dbfile.exists()  # Connecting to non-existent file will create the file, so cache this first
        db = sqlite3.connect(dbfile)
        db.row_factory = sqlite3.Row

        if db_exists:
            fortran_max_zones_existing, zone_system_existing = MatrixButler._preload(db)
            existing_is_compatible = fortran_max_zones == fortran_max_zones_existing and zone_system.equals(
                zone_system_existing)
            if not existing_is_compatible:
                msg = f'Existing matrix cache not compatible with current zone system and will be cleared of any ' \
                      f'stored matrix files. Cache directory is `{parent_directory.as_posix()}`'
                warn(ButlerOverwriteWarning(msg))

                for fp in butler_path.glob(f'*{MatrixButler._MATRIX_EXTENSION}'):
                    fp.unlink()
                MatrixButler._clear_tables(db)
                MatrixButler._create_tables(db, zone_system, fortran_max_zones)
            # When the db exits AND is compatible, there's no need to create the tables
        else:
            MatrixButler._create_tables(db, zone_system, fortran_max_zones)

        return MatrixButler(butler_path, db, zone_system, fortran_max_zones)

    @staticmethod
    def _preload(db: sqlite3.Connection) -> Tuple[int, pd.Index]:
        sql = """
        SELECT *
        FROM properties
        WHERE name="max_zones_fortran"
        """
        result = list(db.execute(sql))
        fortran_max_zones = int(result[0]['value'])

        sql = """
        SELECT *
        FROM zone_system
        """
        result = list(db.execute(sql))
        zone_system = pd.Index([int(record['zone']) for record in result], dtype=np.int64)

        return fortran_max_zones, zone_system

    @staticmethod
    def _clear_tables(db: sqlite3.Connection):
        db.execute("DROP TABLE IF EXISTS properties;")
        db.execute("DROP TABLE IF EXISTS zone_system;")
        db.execute("DROP TABLE IF EXISTS matrices;")
        db.commit()

    @staticmethod
    def _create_tables(db: sqlite3.Connection, zone_system: pd.Index, fortran_max_zones: int):
        sql = """
        CREATE TABLE properties(
        name VARCHAR NOT NULL PRIMARY KEY,
        value VARCHAR
        );
        """
        db.execute(sql)

        sql = """
        INSERT INTO properties
        VALUES (?, ?)
        """
        db.execute(sql, ('max_zones_fortran', fortran_max_zones))
        db.execute(sql, ('zone_partition', -1))

        sql = """
        CREATE TABLE zone_system(
        number INT NOT NULL PRIMARY KEY,
        zone INT
        );
        """
        db.execute(sql)

        sql = """
        INSERT INTO zone_system
        VALUES (?, ?)
        """
        for i, zone in enumerate(zone_system):
            zone = int(zone)  # Cast to Python INT, because SQLite doesn't like NumPy integers.
            db.execute(sql, (i, zone))

        sql = """
        CREATE TABLE matrices(
        id VARCHAR NOT NULL PRIMARY KEY,
        description VARCHAR,
        timestamp VARCHAR,
        type VARCHAR
        );
        """
        db.execute(sql)

        sql = """
        CREATE TABLE matrix_numbers(
        id VARCHAR NOT NULL,
        slice INT NOT NULL,
        number INT,
        PRIMARY KEY (id, slice)
        );
        """
        db.execute(sql)

        db.commit()

    @staticmethod
    def connect(parent_directory: Union[str, Path]) -> MatrixButler:
        """Connect to an existing MatrixButler, without initializing it

        Args:
            parent_directory (Union[str, Path]): The parent directory in which to find the MatrixButler.

        Returns:
            MatrixButler instance.

        Raises:
            IOError: if a MatrixButler cannot be found at the given parent directory.
        """
        parent_directory = Path(parent_directory)

        butler_path = parent_directory / MatrixButler._SUBDIRECTORY_NAME
        if not butler_path.exists():
            raise IOError(f'No matrix butler found at `{parent_directory.as_posix()}`')

        dbfile = butler_path / MatrixButler._DB_NAME
        if not dbfile.exists():
            raise IOError(f'No matrix butler found at `{parent_directory.as_posix()}`')

        db = sqlite3.connect(dbfile)
        db.row_factory = sqlite3.Row
        fortran_max_zones, zone_system = MatrixButler._preload(db)

        return MatrixButler(butler_path, db, zone_system, fortran_max_zones)

    def __iter__(self):
        sql = """
        SELECT *
        FROM matrices
        """
        result = list(self._connection.execute(sql))
        for item in result:
            yield MatrixEntry(item['id'], item['description'], item['timestamp'], item['type'])

    def __contains__(self, item):
        sql = """
        SELECT id
        FROM matrices
        WHERE id=?
        """
        result = list(self._connection.execute(sql, [item]))
        return len(result) > 0

    def __len__(self):
        sql = """
        SELECT *
        FROM matrices
        """
        return len(list(self._connection.execute(sql)))

    def __getitem__(self, item):
        sql = """
        SELECT id
        FROM matrices
        WHERE id=?
        """
        result = list(self._connection.execute(sql, [item]))
        if len(result) < 1:
            raise KeyError(item)

        item = result[0]
        return MatrixEntry(item['id'], item['description'], item['timestamp'], item['type'])

    def to_frame(self) -> pd.DataFrame:
        warn('This method is deprecated. Please use `MatrixButler.list_matrices()` instead')
        return self.list_matrices()

    def list_matrices(self) -> pd.DataFrame:
        """Returns a representation of the butler's contents as a pandas DataFrame"""
        uids, descriptions, timestamps, types = [], [], [], []
        for entry in self:
            uids.append(entry.uid)
            descriptions.append(entry.description)
            timestamps.append(entry.timestamp)
            types.append(entry.type_name)
        df = pd.DataFrame(index=uids)
        df.index.name = 'uid'
        df['description'] = descriptions
        df['timestamp'] = timestamps
        df['type_name'] = types

        return df

    @property
    def zone_system(self) -> pd.Index:
        return self._zone_system[...]  # Ellipses to make a shallow copy

    @property
    def zone_partition(self) -> Optional[int]:
        sql = """
        SELECT *
        FROM properties
        WHERE name="zone_partition"
        """
        result = list(self._connection.execute(sql, ''))
        zone_partition = int(result[0]['value'])
        return None if zone_partition <= 0 else zone_partition

    @zone_partition.setter
    def zone_partition(self, val):
        if val is not None:
            val = int(val)
            assert val in self._zone_system
        else:
            val = -1
        sql = """
        INSERT OR REPLACE INTO properties
        VALUES (?, ?)
        """
        self._connection.execute(sql, ('zone_partition', val))
        if self._committing:
            self._connection.commit()

    def _matrix_file(self, n: int) -> Path:
        return self._path / f'mf{n}{MatrixButler._MATRIX_EXTENSION}'

    @contextmanager
    def batch_operations(self) -> Generator[None, None, None]:
        """Context-manager for writing several matrices in one batch. Reduces write time per matrix by committing
        changes to the DB at the end of the batch write. The time savings can be quite significant, as the DB-write is
        normally 50% of the time per matrix write."""
        # This snippet is just in case this function is called within its own context (e.g. someone turns on
        # batch mode while it's already on).
        if not self._committing:
            yield
            return

        self._committing = False

        try:
            yield
        finally:
            self._connection.commit()
            self._committing = True

    def lookup_numbers(self, unique_id: str, *, squeeze: bool = True) -> Union[int, List[int]]:
        """Looks up file number(s) (e.g. 50 corresponds to "mf50.bin") of a given matrix.

        Args:
            unique_id (str): The ID of the matrix to look up
            squeeze (bool): If True, and only one matrix number corresponds to the unique ID, then the result will
                be a single integer. Otherwise, a list of integers is returned.

        Returns:
            Union[int, List[int]], depending on results and `squeeze`
        """
        sql = """
        SELECT *
        FROM matrix_numbers
        WHERE id=?
        """
        result = list(self._connection.execute(sql, [unique_id]))
        if not result:
            raise KeyError(unique_id)

        numbers = [int(item['number']) for item in result]

        if len(numbers) == 1 and squeeze:
            numbers = numbers[0]
        return numbers

    def is_sliced(self, unique_id: str) -> bool:
        """Checks if a matrix is sliced on-disk or not.

        Args:
            unique_id (str): The ID of the matrix to check

        Returns:
            bool: True if matrix is sliced, False otherwise.

        Raises:
            KeyError: if unique_id is not in the butler.
        """
        return len(self.lookup_numbers(unique_id, squeeze=False)) > 1

    def _next_numbers(self, n: int) -> List[int]:
        sql = "SELECT number FROM matrix_numbers;"
        existing_numbers = {int(item['number']) for item in list(self._connection.execute(sql))}

        numbers = []
        cursor = 1
        for _ in range(n):
            while cursor in existing_numbers:
                cursor += 1
            numbers.append(cursor)
            cursor += 1

        return numbers

    def _check_lookup(self, unique_id: str, n: int, partition: bool) -> List[int]:
        if partition:
            n += 1

        try:
            numbers = self.lookup_numbers(unique_id, squeeze=False)
            if len(numbers) != n:
                # Overwriting an existing matrix with different number of slices causes that matrix to be completely
                # replaced (deleted and then overwritten)
                self.delete_matrix(unique_id)
                numbers = self._next_numbers(n)
        except KeyError:
            numbers = self._next_numbers(n)

        return numbers

    def _write_matrix_record(self, unique_id: str, numbers: List[int], description: str, type_name: str):
        timestamp = dt.now()
        sql = """
        INSERT OR REPLACE INTO matrices
        VALUES (?, ?, ?, ?)
        """
        self._connection.execute(sql, (unique_id, description, timestamp, type_name))

        for slice_, number in enumerate(numbers):
            sql = """
            INSERT OR REPLACE INTO matrix_numbers
            VALUES (?, ?, ?)
            """
            self._connection.execute(sql, (unique_id, slice_, number))

        if self._committing:
            self._connection.commit()

    def _validate_slice_args(self, n_slices: int, partition: bool) -> Tuple[int, bool]:
        n_slices = int(n_slices)
        assert n_slices >= 1

        if partition and self.zone_partition is None:
            warn("Cannot partition a matrix when MatrixButler.zone_partition is None.")
            return n_slices, False

        return n_slices, partition

    # def _expand_matrix(self, matrix, n_slices, partition):
    #     if n_slices == 1 and not partition:
    #         rows, cols = matrix.shape
    #         assert rows == cols
    #         padding = self._max_zones_fortran - rows
    #         if padding > 0:
    #             return expand_array(matrix, padding, axis=None)
    #         return matrix
    #     else:
    #         cols = matrix.shape[1]
    #         padding = self._max_zones_fortran - cols
    #         if padding > 0:
    #             return expand_array(matrix, padding, axis=1)
    #     return matrix

    def _write_matrix_files(self, matrix_array: np.ndarray, files: List[Path], partition: bool):

        # matrix = self._expand_matrix(matrix, len(files), partition)

        remainder, remainder_file = None, None
        if partition:
            slice_end = self._zone_system.get_loc(self.zone_partition) + 1

            remainder = matrix_array[slice_end:, :]
            remainder_file = files.pop()  # Remove from the list of files

            matrix_array = matrix_array[:slice_end, :]

        n_slices = len(files)
        slices = np.array_split(matrix_array, n_slices, axis=0) if n_slices > 1 else [matrix_array]

        min_index = 1
        for slice_, file_ in zip(slices, files):
            self._to_binary_file(slice_, file_, min_index)
            min_index += slice_.shape[0]
        if partition:
            self._to_binary_file(remainder, remainder_file, min_index)

    def init_matrix(self, unique_id: str, *, description: str = "", type_name: str = "", fill: bool = True,
                    n_slices: int = 1, partition: bool = False):
        """Registers a new (or zeros an old) matrix with the butler.

        Args:
            unique_id (str): The unique identifier for this matrix.
            description (str):  A brief description of the matrix.
            type_name (str): Type categorization of the matrix.
            fill (bool): If False, empty (0-byte) files will be initialized. Otherwise, 0-matrix files will be created.
            n_slices (int): Number of slices (on-disk) for multi-processing.
            partition (bool): Flag whether to partition the matrix before saving, based on self.zone_partition
        """
        n_slices, partition = self._validate_slice_args(n_slices, partition)

        numbers = self._check_lookup(unique_id, n_slices, partition)
        files = [self._matrix_file(n) for n in numbers]

        if fill:
            shape = [len(self._zone_system)] * 2
            matrix = np.zeros(shape, dtype=np.float32)
            self._write_matrix_files(matrix, files, partition)

        self._write_matrix_record(unique_id, numbers, description, type_name)

    def load_matrix(self, unique_id: str, *, tall: bool = False) -> Union[pd.DataFrame, pd.Series]:
        """Gets a matrix from the butler, optionally saving into an Emmebank.

        Args:
            unique_id (str): The name you gave to the butler for safekeeping.
            tall (bool): Defaults to ``False``. A flag to return the matrix as a stacked `DataFrame` (i.e. Series)

        Returns:
            DataFrame or Series, depending on whether `tall` is given.

        Raises:
            KeyError: if unique_id is not in the butler.
        """

        is_sliced = self.is_sliced(unique_id)
        numbers = self.lookup_numbers(unique_id, squeeze=False)

        matrices = []
        for number in numbers:
            fp = self._matrix_file(number)
            subframe = self._from_binary_file(fp)
            matrices.append(subframe)
        matrix = pd.concat(matrices, axis=0) if is_sliced else matrices[0]

        if not matrix.index.equals(self._zone_system):
            matrix = matrix.reindex(self._zone_system, fill_value=0.0)

        if tall:
            return matrix.stack()
        return matrix

    def save_matrix(self, dataframe_or_array: Union[pd.DataFrame, np.ndarray], unique_id: str, *, description: str = "",
                    type_name: str = "", n_slices: int = 1, partition: bool = False, reindex: bool = True,
                    fill_value: float = 0.0):
        """Passes a matrix to the butler for safekeeping.

        Args:
            dataframe_or_array (Union[pd.DataFrame, np.ndarray]): Specifies the matrix to save. Must be square in shape.
            unique_id (str): The unique identifier for this matrix.
            description (str): A brief description of the matrix.
            type_name (str): The string type
            n_slices (int): Number of slices (on-disk) for multi-processing.
            partition (bool): Flag whether to partition the matrix before saving, based on self.zone_partition
            reindex (bool): Flag to indicate if partial matrices are accepted when supplying a DataFrame. If False,
                AssertionError will be raised when one of the DataFrame's axes doesn't match the Butler's zone system.
            fill_value (float): The fill value to be used when reindexing (see 'reindex' flag)
        """

        n_slices, partition = self._validate_slice_args(n_slices, partition)
        matrix = self._coerce_matrix(dataframe_or_array, reindex=reindex, fill_value=fill_value)

        numbers = self._check_lookup(unique_id, n_slices, partition)
        files = [self._matrix_file(n) for n in numbers]
        self._write_matrix_files(matrix, files, partition)

        self._write_matrix_record(unique_id, numbers, description, type_name)

    def query_type(self, type_name: str) -> List[str]:
        """Gets a list of matrix IDs by their type name

        Args:
            type_name (str): The type name to query

        Returns:
            A list of matching IDs
        """
        sql = """
        SELECT id FROM matrices
        WHERE type=?;
        """
        return [item['id'] for item in self._connection.execute(sql, [type_name])]

    def matrix_metadata(self, unique_id: str) -> Dict[str, Any]:
        """Looks up a single matrix record and returns its metadata (description, type, timestamp) in a dictionary

        Args:
            unique_id (str): The unique ID of the matrix to lookup

        Returns:
            Dict: corresponding to the matrix record. Keys are 'id', 'description', 'timestamp', and 'type'

        Raises:
            KeyError: if unique ID not in the butler.
        """

        sql = """
        SELECT id, description, type, timestamp FROM matrices
        WHERE id=?;
        """
        results = list(self._connection.execute(sql, [unique_id]))
        if not results:
            raise KeyError(unique_id)
        row = results[0]
        return dict(row)

    def delete_matrix(self, unique_id: str):
        """Deletes a matrix from the butler's directory

        Args:
            unique_id (str): The unique identifier of the matrix to delete.

        Raises:
            KeyError: if unique_id cannot be found.
        """

        numbers = self.lookup_numbers(unique_id, squeeze=False)
        for n in numbers:
            fp = self._matrix_file(n)
            if fp.exists():
                fp.unlink()

        sql = """
        DELETE FROM matrices
        WHERE id=?;
        """
        self._connection.execute(sql, [unique_id])

        sql = """
        DELETE FROM matrix_numbers
        WHERE id=?;
        """
        self._connection.execute(sql, [unique_id])

        if self._committing:
            self._connection.commit()

    def __del__(self):
        self._connection.commit()
        self._connection.close()

    def slice_matrix(self, unique_id: str, *, n_slices: int = 1, partition: bool = None):
        """Slices a matrix into chunks along its rows (on-disk) for use in multi-processing. Does nothing if matrix is
        already sliced.

        Args:
            unique_id (str): The ID of the matrix to slice
            n_slices (int): The number of slices to make
            partition (bool): Flag whether to partition the matrix before saving, based on `self.zone_partition`
        """
        prior_n_slices = len(self.lookup_numbers(unique_id, squeeze=False))
        expected_slices = n_slices + 1 if partition else n_slices
        if prior_n_slices == expected_slices:
            return  # Do nothing if already sliced to the called number

        n_slices, partition = self._validate_slice_args(n_slices, partition)

        matrix: pd.DataFrame = self.load_matrix(unique_id, tall=False)
        metadata = self.matrix_metadata(unique_id)

        with self.batch_operations():
            self.delete_matrix(unique_id)
            self.save_matrix(matrix, unique_id, metadata['description'], metadata['type'], n_slices=n_slices,
                             partition=partition)

    def unslice_matrix(self, unique_id: str):
        """Un-slices (i.e. concatenates) a sliced matrix in the butler. Does nothing if the matrix is not sliced.

        Args:
            unique_id (str): The ID of the matrix to slice.
        """
        if not self.is_sliced(unique_id):
            return  # Do nothing is matrix is already not sliced
        matrix: pd.DataFrame = self.load_matrix(unique_id, tall=False)
        metadata = self.matrix_metadata(unique_id)

        with self.batch_operations():
            self.delete_matrix(unique_id)
            self.save_matrix(matrix, unique_id, metadata['description'], metadata['type'], n_slices=1)

    # region matrices

    def _to_binary_file(self, array: np.ndarray, file_: Path, min_index: int = 1):
        assert min_index >= 1

        rows, columns = array.shape

        # Mask the integer binary representation as floating point
        index = np.arange(min_index, min_index + rows, dtype=np.int32)
        index_as_float = np.frombuffer(index.tobytes(), dtype=np.float32).reshape(rows, 1)

        to_concat = [index_as_float, array]
        extra_columns = self._max_zones_fortran - columns
        if extra_columns > 0:
            padding = np.zeros(shape=(rows, extra_columns), dtype=np.float32)
            to_concat.append(padding)

        expanded = np.concatenate(to_concat, axis=1)

        with open(file_, mode='wb') as writer:
            expanded.tofile(writer)

    def _from_binary_file(self, file_: Path) -> pd.DataFrame:
        with open(file_, mode='rb') as reader:
            raw_floats = np.fromfile(reader, dtype=np.float32)

        rows = len(raw_floats) // (self._max_zones_fortran + 1)
        assert len(raw_floats) == (rows * (self._max_zones_fortran + 1))

        raw_floats.shape = rows, self._max_zones_fortran + 1
        row_offsets = np.frombuffer(raw_floats[:, 0].tobytes(), dtype=np.int32) - 1
        row_index = self.zone_system.take(row_offsets[: len(self.zone_system)])  # MLOGIT likes to pad out to max zones, which we don't need.

        real_columns = len(self.zone_system)
        matrix = raw_floats[: len(self.zone_system):, 1: real_columns + 1].copy()  # Drop extra columns and make a deep copy to force GC to cleanup the raw

        frame = pd.DataFrame(matrix, index=row_index, columns=self.zone_system)
        return frame

    def _coerce_matrix(self, data: Union[pd.DataFrame, np.ndarray], reindex: bool = True,
                       fill_value: float = 0.0) -> np.ndarray:
        if isinstance(data, pd.DataFrame):
            if not data.index.equals(self.zone_system):
                if not reindex:
                    raise AssertionError()
                data: pd.DataFrame = data.reindex(self.zone_system, axis=0, fill_value=fill_value)
            if not data.columns.equals(self.zone_system):
                if not reindex:
                    raise AssertionError()
                data: pd.DataFrame = data.reindex(self.zone_system, axis=1, fill_value=fill_value)
            return data.to_numpy(copy=True).astype(np.float32)
        elif isinstance(data, np.ndarray):
            i, j = data.shape
            assert i == j, "Non-square raw matrices are not supported"
            assert i == len(self.zone_system), "Matrices must be the same length at the Butler zone system"
            return data.astype(np.float32)
        raise NotImplementedError(type(data))

        # endregion


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):
    dependencies = [
        ('apptest', '0002_auto_20150318_1813'),
    ]

    operations = [
        migrations.AddField(
            model_name='customer',
            name='address',
            field=models.CharField(max_length=100, blank=True),
            preserve_default=True,
        ),
        #Populate the field address in the db with the name of the user + '- No address provided'
        migrations.RunSQL("UPDATE apptest_customer SET address=name || '- No address provided';")
    ]


import pandas as pd
import pdb
import numpy as np
import random
import math
import argparse
import tensorflow as tf
import matplotlib.pyplot as plt
from keras import losses
from keras import models
from keras import regularizers
from keras.layers.core import Dense
from keras.layers.core import Dropout
from keras.layers.core import Flatten
from keras.layers.recurrent import LSTM
from keras.layers.recurrent import GRU
from keras.layers.convolutional import Conv1D
from keras.callbacks import TensorBoard
from keras.optimizers import adam
from keras.layers.advanced_activations import LeakyReLU
from shutil import copyfile

INPUT_COUNT = 40

def loadModel(model):
    try:
        model.load_weights("model.h5")
        print("model loaded")
    except OSError:
        print("can't find model file to load")

def saveModel(model):
    print("Saving model")
    try:
        copyfile('model.h5', 'model.h5.saved')
    except FileNotFoundError:
        print("didn't find model files to save")

    model.save_weights("model.h5", overwrite=True)

# generate a sequence of data, using x = sin(a*i). Passing in a and count
def getSineData(a, count):
    return np.array([math.sin(a * i) for i in range(0, count)])

# Given the time series data and data size, return the x,y data suitable for training
# x - time lagged by INPUT_COUNT, so for each time step we have INPUT_COUNT number of data points
# y - for each data point x, the data at the next time step
def getXY(data, total):
    x = data[0: total - INPUT_COUNT].reshape((total - INPUT_COUNT, 1))
    for i in range(1, INPUT_COUNT):
        x = np.concatenate((x, data[i: total - INPUT_COUNT + i].reshape((total - INPUT_COUNT, 1))), axis=1)

    y = data[INPUT_COUNT : total]
    return x,y

# Generate the training and evaluating data sets for data series sin(a*x)
def getSeries(a, trainCount, testCount=0):
    data = getSineData(a, trainCount+testCount)
    x,y = getXY(data, trainCount)
    if testCount > INPUT_COUNT:
        evalX, evalY = getXY(data[trainCount:], testCount)
    else:
        evalX, evalY = None, None
    return x, y, evalX, evalY

def getCNNSeries(a, trainCount, testCount):
    x, y, evalX, evalY = getSeries(a, trainCount, testCount)
    if testCount <= INPUT_COUNT:
        return x.reshape(x.shape[0], x.shape[1], 1), y, None, None
    return x.reshape(x.shape[0], x.shape[1], 1), y, evalX.reshape(evalX.shape[0], evalX.shape[1], 1), evalY

def buildFCModel():
    model = models.Sequential()
    model.add(Dense(100, input_shape=(INPUT_COUNT,)))
    model.add(LeakyReLU(alpha=0.03))
    model.add(Dense(100))
    model.add(LeakyReLU(alpha=0.03))
    model.add(Dense(1))

    model.compile(loss='mse', optimizer=adam(lr=0.0001))

    board = TensorBoard(log_dir='model', histogram_freq=1, write_graph=True, write_images=False)
    board.set_model(model)
    return model, board

def buildCNNModel():
    model = models.Sequential()
    model.add(Conv1D(100, 3, strides=1, input_shape=(INPUT_COUNT, 1)))
    model.add(LeakyReLU(alpha=0.03))
    model.add(Conv1D(100, 3, strides=1))
    model.add(LeakyReLU(alpha=0.03))
    model.add(Flatten())
    model.add(Dense(100))
    model.add(LeakyReLU(alpha=0.03))
    model.add(Dense(1))

    model.compile(loss='mse', optimizer=adam(lr=0.0001))

    board = TensorBoard(log_dir='model', histogram_freq=1, write_graph=True, write_images=False)
    board.set_model(model)
    return model, board

def buildLSTMModel():
    model = models.Sequential()
    model.add(LSTM(100, batch_input_shape=(1, 1, 1), return_sequences=True, stateful=True))
    model.add(LSTM(100, return_sequences=False, stateful=True))
    model.add(Dense(1))
    model.compile(loss='mse', optimizer=adam(lr=0.0001))

    board = TensorBoard(log_dir='model', histogram_freq=1, write_graph=True, write_images=False)
    board.set_model(model)
    return model, board

def testFC():
    model, board = buildFCModel()

    allX, allY, allEX, allEY = getSeries(0.06, 3000, 300)

    for i in range(1, 10):
        x, y, evalX, evalY = getSeries(0.06 + i * 0.006, 3000, 300)
        allX = np.concatenate((allX, x))
        allY = np.concatenate((allY, y))
        allEX = np.concatenate((allEX, evalX))
        allEY = np.concatenate((allEY, evalY))

    history = model.fit(x=allX, y=allY, validation_data=(allEX, allEY), batch_size=100, epochs=10, callbacks=[board])

    plotGenerated(model, 0.06)
    plotGenerated(model, 0.083)
    plotGenerated(model, 0.163)
    plotGenerated(model, 0.033)
    return

def testCNN():
    model, board = buildCNNModel()

    allX, allY, allEX, allEY = getCNNSeries(0.06, 3000, 300)

    for i in range(1, 10):
        x, y, evalX, evalY = getCNNSeries(0.06 + i * 0.006, 3000, 300)
        allX = np.concatenate((allX, x))
        allY = np.concatenate((allY, y))
        allEX = np.concatenate((allEX, evalX))
        allEY = np.concatenate((allEY, evalY))

    history = model.fit(x=allX, y=allY, validation_data=(allEX, allEY), batch_size=100, epochs=10, callbacks=[board])

    plotGeneratedCNN(model, 0.06)
    plotGeneratedCNN(model, 0.083)
    plotGeneratedCNN(model, 0.163)
    plotGeneratedCNN(model, 0.033)
    return

def testLSTM():
    model, board = buildLSTMModel()

    total = 3300
    trainSize = 3000

    loadModel(model)

    for epoch in range(0, 10):
        for i in range(0, 10):
            data = getSineData(0.06 + i * 0.006, total)
            x = data[0:trainSize].reshape((trainSize, 1, 1))
            y = data[1:trainSize+1]
            evalX = data[trainSize:total-1].reshape((total-1-trainSize, 1, 1))
            evalY = data[trainSize+1:total]
            model.fit(x=x, y=y, validation_data=(evalX, evalY), batch_size=1, epochs=1, shuffle=False)
            model.reset_states()

    saveModel(model)

    plotGeneratedLSTM(model, 0.06)
    plotGeneratedLSTM(model, 0.083)
    plotGeneratedLSTM(model, 0.163)
    plotGeneratedLSTM(model, 0.033)
    return

def render(y, predicted, a, count):
    plt.grid()
    plt.plot(y[0:count])
    plt.plot(predicted[0:count], 'go')
    plt.title("y=sin({}*x)".format(a))
    plt.savefig("plt-{}.png".format(a))
    plt.show()

# Create a series of data points using parameter a (as in sin(a*i)). Predict y using the real
# values of x for each data point. Plot out y against predicted data points.
def plotPredicted(model, a):
    model.reset_states()
    x, y, eX, eY = getSeries(a, 300)
    predicted = model.predict(x=x)
    render(y, predicted, a, 100)

# Start a series of data using parameter a (as in sin(a*i)). Using the data as starting point,
# predict the next data in the time series. Then predict more using the generated data points
# form earlier predictions.
def plotGenerated(model, a):
    x, y, eX, eY = getSeries(a, 300)
    predicted = np.zeros((100,))
    gx = x[0].copy()
    for i in range(0, 100):
        predicted[i] = model.predict(x=np.array([gx]))
        gx = np.append(gx[1:], predicted[i])

    render(y, predicted, a, 100)

def plotGeneratedCNN(model, a):
    x, y, eX, eY = getCNNSeries(a, 300, 0)
    predicted = np.zeros((100,))
    gx = x[0].copy()
    for i in range(0, 100):
        predicted[i] = model.predict(x=np.array([gx]))
        gx = np.append(gx[1:], np.array([[predicted[i]]]), axis=0)

    render(y, predicted, a, 100)

def plotGeneratedLSTM(model, a):
    model.reset_states()
    data = getSineData(a, 300)
    x = data[0:100]
    # Build up LSTM state
    model.predict(x=x.reshape((100, 1, 1)), batch_size=1)

    y = data[101:201]
    predicted = np.zeros((100,))
    gx = data[0]
    for i in range(100, 200):
        predicted[i-100] = model.predict(x=gx.reshape(1, 1, 1))
        gx = predicted[i-100]

    render(y, predicted, a, 100)

def main():
    parser = argparse.ArgumentParser(description='Trying to model unseen sine function')
    parser.add_argument("-m", "--model", help="fc/cnn/lstm")

    args = parser.parse_args()
    if args.model == 'fc':
        testFC()
    elif args.model == 'cnn':
        testCNN()
    elif args.model == 'lstm':
        testLSTM()
    else:
        print('unknown model')
    return

if __name__ == '__main__':
    main()


import numpy as np
from matplotlib import rc
import matplotlib.pyplot as plt

x_0 = np.linspace(0, np.pi, 100) #the x mesh
n = 10 #number in the sum of the fourier series

n_list = np.arange(1,n+1, 1) # an array of integers from 1 to n

#An array of the fourier coefficients. The RHS will have to be updated for every series
A_n_list = (2 / n_list)*((-1)**(n_list))

# A python list to append to. The internet says this is faster than directly declaring
# a matrix of arrays using numpy
sin_list = []

# Appends arrays to a list. Each row is sin(nx), one row for each n.
# Again this would have to be updated for every fourier series
for i in range(0,n,1):
    sin_list.append(np.sin(n_list[i] * x_0))

#Turns the list of arrays into a numpy matrix
sin_matrix = np.asarray(sin_list)

A_column = A_n_list.reshape(-1, 1) #reshape coefficients for multiplication

#Multiplies each coefficient across the entire corresponding x-mesh
#then sums the columns of the matrix to get the final value u(x, 0).
u = np.sum(A_column*sin_matrix, 0)
#     u = A_n * np.cos(n * t) * np.sin(n*x_0)
#
#     return u

#u_0 = fsoln(n_list, A_n_list, 0)  # type: float

plt.plot(x_0, u, 'r')
plt.axis([0, np.pi, -np.pi, np.pi])
plt.show()


from redis import Redis
from rq import SimpleWorker, Queue
from llib import DogFetch
from birdDog import findbird
from birdDog import speak
from birdDog import chewbird
from llib.cmdutil import cmdutil
import click
import os

def myip():
    myip = cmdutil.subprocess_cmd('ec2metadata --local-ipv4')
    return(myip)

def findTheBird(dataSet, entityId, productId, redisServer):
    redisURL='redis://' + redisServer
    queue = Queue('bird', connection=Redis.from_url(redisURL))
    queue.enqueue(findbird, dataSet, entityId, productId, timeout=1800)

def pleaseSpeak(requestor, redisServer):
    redisURL='redis://' + redisServer
    queue = Queue('bird', connection=Redis.from_url(redisURL))
    queue.enqueue(speak, requestor, timeout=1800)

def chewTar(bucket, tarfile, redisServer):
    redisURL='redis://' + redisServer
    queue = Queue('bird', connection=Redis.from_url(redisURL))
    queue.enqueue(chewbird, bucket, tarfile, timeout=1800)

def huntEmUp(path,row, product):
    """ this gets a path row for a given product/dataset """
    pup = DogFetch('dogTrainer')

    entlist = pup.hunt(path=path, row=row, product=product)
    redis = pup.redis
    #redis = myip()
    pup.logi('redis = %s' % redis)
    print('redis = %s' % redis)
    cnt=0;
    for it in entlist:
        print(it)
        myid = it[0]
        myprod = it[1]
        if (cnt < 24):
            findTheBird(dataSet = product, entityId=myid, productId=myprod, redisServer=redis)
        cnt = cnt + 1

    pup.logi('Count = %s' % cnt)

def get_todo_list(bucket):
    print (bucket)
    print (bucket)
    filelist= []
    print (' scene list initializing ...')
    cmd = 'aws s3 ls --recursive %s/tar > /data/listings/tarlist.txt' % bucket
    os.system(cmd)
    file = open('/data/listings/tarlist.txt')
    for line in file:
        line = line.rstrip()
        if 'tar/' in line:
            tarfile = line.split()[3]
            filelist.append(tarfile)

    return (filelist)

def chewEmUp(bucket):
    pup = DogFetch('dogTrainer')
    redis = pup.redis
    bucket = pup.cog_bucket
    #pup.logi('chewEmUp, redis = %s' % redis)
    #pleaseSpeak('dogTrainer-chewEmUp', redis)
    list = get_todo_list(bucket)
    cnt=0;
    for item in list:
        print(item)
        if (cnt < 24):
            chewTar(bucket, item, redis)
            cnt = cnt + 1

@click.group()
def cli():
    """this script runs a pack of dogs -
    retrievers  get a bunch of tar files from EarthExplorer
    policeDogs chew these tar balls up and make cogs
    all of this goes in the cloud in an S3 bucket"""
    pass

@cli.command()
@click.option('--path', default='46', help="WRS PATH (path)", required=True)
@click.option('--row', default='28', help="WRS ROW (row)", required=True)
@click.option('--product', default='LANDSAT_ETM_C1', help=" other codes here LANDSAT_ETM_C1 (product)", required=True)
def retrieve(path, row, product):
    """ find a list of files and start the pack of retrievers via the job queue"""

    print ("retrieve started ")
    huntEmUp(path,row, product)

@cli.command()
@click.option('--bucket', default='CHSBUCKETHERE', help="This is the name of the AWS S3 Bucket (bucket)", required=True)

def makecogs(bucket):
    """ make cogs - this takes a bucket of tarballs and makes cogs via a pack of policeDogs"""

    print ("makecogs started ")
    chewEmUp(bucket)