#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Created on Jan 10, 2013

@author: daniel

This utility file provides a method for moving information between Sugar
instances.
'''

from collections import defaultdict
from sugarcrm import Sugarcrm
from code import interact
from sugarcrm.sugarentry import SugarEntry

com_url = 'http://<host>:<port>/service/v4_1/rest.php'
com_usr = '<user>'
com_pwd = '<pass>'

pro_url = 'http://<host>:<port>/service/v4_1/rest.php'
pro_usr = '<user>'
pro_pwd = '<pass>'

modules = ['Accounts', 'Contacts', 'Opportunities', 'Leads', 'Notes',
           'Prospects', 'Tasks']

cache = defaultdict(dict)
# Fill in user values from one system to the other here.
#   The mapping is From -> To.  Ie: Old System -> New System.
cache['Users']['1'] = '1'

# This map holds fields that need to pull from other cached values.
relate = {
  'Contacts': {
    'account_id': 'Accounts',
    'assigned_user_id': 'Users'
  },
  'Opportunities': {
    'account_id': 'Accounts',
    'assigned_user_id': 'Users'
  },
  'Leads': {
    'account_id': 'Accounts',
    'assigned_user_id': 'Users',
    'contact_id': 'Contacts',
    'opportunity_id': 'Opportunities',
  },
  'Prospects': {
    'assigned_user_id': 'Users',
    'lead_id': 'Leads'
  },
  'Tasks': {
    'assigned_user_id': 'Users',
    'contact_id': 'Contacts'
  },
  'Calls': {
    'assigned_user_id': 'Users',
    'contact_id': 'Contacts'
  },
  'Notes': {
    'account_id': 'Accounts',
    'assigned_user_id': 'Users',
    'contact_id': 'Contacts',
    'lead_id': 'Leads',
    'opportunity_id': 'Opportunities'
  },
  'Accounts': {
    'assigned_user_id': 'Users'
  }
}

SPro = Sugarcrm(pro_url, pro_usr, pro_pwd)
SCom = Sugarcrm(com_url, com_usr, com_pwd)

# A second lookup, this one for required module level connections that
# must be generated.
mod_links = {
  'Tasks': [SCom.modules['Leads'],
            SCom.modules['Notes'],
            SCom.modules['Opportunities'],
            SCom.modules['Accounts']],
  'Notes': [SCom.modules['Opportunities'],
            SCom.modules['Leads'], ]
}

def makeProEntry(oldEntry, oldID = None):
  module = oldEntry._module
  mod_name = module._name
  newEntry = SugarEntry(SPro, mod_name)
  for field in module._fields.keys():
    if field == 'id':
      oldID = oldEntry[field]
      continue
    if field in relate[mod_name]:
      ref_mod = relate[mod_name][field]
      newEntry[field] = cache[ref_mod].get(oldEntry[field], '')
      continue
    newEntry[field] = oldEntry[field]
  newEntry.save()
  for relmod in mod_links.get(mod_name, []):
    for relentry in oldEntry.get_related(relmod):
      if relentry['id'] in cache[relmod._name]:
        newrelentry = SPro.newSugarEntry(relmod._name)
        newrelentry['id'] = cache[relmod._name][relentry['id']]
        newEntry.relate(newrelentry)
  if oldID is not None:
    cache[mod_name][oldID] = newEntry['id']

if __name__ == '__main__':
  interact(local = globals())


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 15 20:50:05 2022

@author: Yifang
"""
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import traceAnalysis as Ananlysis
import SPADdemod

def getSignalTrace (filename, traceType='Constant',HighFreqRemoval=True,getBinTrace=False,bin_window=20):
    '''TraceType:Freq, Constant, TimeDiv'''
    trace=Ananlysis.Read_trace (filename,mode="SPAD")
    if HighFreqRemoval==True:
        trace=Ananlysis.butter_filter(trace, btype='low', cutoff=1000, fs=9938.4, order=10)
    if traceType=='Constant':
        if getBinTrace==True:
            trace_binned=Ananlysis.get_bin_trace(trace,bin_window=bin_window,color='m')
            trace_binned=Ananlysis.get_bin_trace(trace,bin_window=bin_window)
            return trace_binned
        else:
            return trace
    if traceType=='Freq':
        #Red,Green= SPADdemod.DemodFreqShift (trace,fc_g=1000,fc_r=2000,fs=9938.4)
        Red,Green= SPADdemod.DemodFreqShift_bandpass (trace,fc_g=1009,fc_r=1609,fs=9938.4)
        #Red=Ananlysis.butter_filter(Red, btype='low', cutoff=200, fs=9938.4, order=10)
        #Green=Ananlysis.butter_filter(Green, btype='low', cutoff=200, fs=9938.4, order=10)
        Signal=Ananlysis.getSignal_subtract(Red,Green,fs=9938.4)
        return Red,Green,Signal
    if traceType=='TimeDiv':
        #need to be modified for different time division traces
        lmin,lmax=SPADdemod.hl_envelopes_max(trace, dmin=2, dmax=2, split=True)
        fig, ax = plt.subplots(figsize=(12, 3))
        ax.plot(lmax,trace[lmax], color='r')
        ax.plot(lmin,trace[lmin], color='g')
        x_green, Green=SPADdemod.Interpolate_timeDiv (lmin,trace)
        x_red, Red=SPADdemod.Interpolate_timeDiv (lmax,trace)

        Signal=Ananlysis.getSignal_subtract(Red,Green,fs=9938.4)
        fig, ax = plt.subplots(figsize=(12, 3))
        ax=Ananlysis.plot_trace(Signal,ax, label="Signal")
        return Red,Green,Signal

#%%
# Sampling Frequency
fs   = 9938.4
#dpath= "C:/SPAD/SPADData/20220611/1516996_Freq_2022_6_11_16_8_21"
dpath="D:/SPAD/SPADData/20220913/1534725_HPC_50g_2022_9_13_16_3_57"

#%%
filename=Ananlysis.Set_filename (dpath,"traceValue.csv")
#Red,Green,Signal=getSignalTrace (filename,traceType='TimeDiv',HighFreqRemoval=True,getBinTrace=False)
Signal_raw=getSignalTrace (filename,traceType='Constant',HighFreqRemoval=True,getBinTrace=False,bin_window=100)
#%%
import traceAnalysis as Ananlysis
bin_window=100

Signal_bin=Ananlysis.get_bin_trace(Signal_raw,bin_window=bin_window)
fig, ax = plt.subplots(figsize=(12, 2.5))
Ananlysis.plot_trace(Signal_bin,ax, fs=99.38, label="Binned to 100Hz",color="b")
#%%
import traceAnalysis as Ananlysis
bin_window=200
Red_bin=Ananlysis.get_bin_trace(Red,bin_window=bin_window)
Green_bin=Ananlysis.get_bin_trace(Green,bin_window=bin_window)
#%%
fig, ax = plt.subplots(figsize=(12, 2.5))
Ananlysis.plot_trace(Red_bin[0:500],ax, fs=49.7, label="Binned to 50Hz",color="r")

#ax.set_xlim([0, 0.1])
fig, ax = plt.subplots(figsize=(12, 2.5))
Ananlysis.plot_trace(Green_bin[0:500],ax, fs=49.7, label="Binned to 50Hz",color="g")
#ax.set_xlim([0, 0.1])
#%%
'''unmixing time division'''
lmin,lg=SPADdemod.hl_envelopes_max(Green, dmin=4, dmax=7, split=True)
lmin,lr=SPADdemod.hl_envelopes_max(Red, dmin=4, dmax=7, split=True)
fig, ax = plt.subplots(figsize=(12, 3))
ax.plot(lg,Green[lg], color='g')
ax.plot(lr,Red[lr], color='r')
x_red, Red=SPADdemod.Interpolate_timeDiv (lr,Red)
x_green, Green=SPADdemod.Interpolate_timeDiv (lg,Green)

#%%
Signal=Ananlysis.getSignal_subtract(Red,Green,fs=49.7)
#%%
fig, (ax0, ax1,ax2) = plt.subplots(nrows=3)
ax0=Ananlysis.plot_trace(Green,ax0, fs=49.7, label="Green Signal 200Hz",color='g')
ax1=Ananlysis.plot_trace(Red,ax1, fs=49.7, label="Red Signal 200Hz", color='r')
ax2=Ananlysis.plot_trace(Signal,ax2, fs=49.7, label="Substract Signal 200Hz", color='b')
fig.tight_layout()
#%%
Signal=Ananlysis.butter_filter(Signal, btype='low', cutoff=100, fs=9938.4, order=5)

fig, ax = plt.subplots(figsize=(12, 2.5))
Ananlysis.plot_trace(Signal,ax, fs=9938.4, label="100Hz Low pass")
#%%
'''temporal bin dual channel'''
bin_window=200
Green_bin=Ananlysis.get_bin_trace(Green,bin_window=bin_window)
Red_bin=Ananlysis.get_bin_trace(Red,bin_window=bin_window)
Signal_binned=Ananlysis.get_bin_trace(Signal,bin_window=bin_window)

fig, (ax0, ax1,ax2) = plt.subplots(nrows=3)
ax0=Ananlysis.plot_trace(Green_bin,ax0, fs=99.384/2, label="Green Signal Binned 50Hz",color='g')
ax1=Ananlysis.plot_trace(Red_bin,ax1, fs=99.384/2, label="Red Signal Binned 50Hz", color='r')
ax2=Ananlysis.plot_trace(Signal_binned,ax2, fs=99.384/2, label="Substract Signal Binned 50Hz", color='b')
fig.tight_layout()

#%%
fs=200
fig, ax = plt.subplots(figsize=(8, 2))
powerSpectrum, freqenciesFound, time, imageAxis = ax.specgram(Signal,Fs=fs,NFFT=1024, detrend='linear',vmin=-130)
ax.set_xlabel('Time (Second)')
ax.set_ylabel('Frequency')
ax.set_ylim([0, 100])

#%%
signal1,signal2=Ananlysis.getICA (Red_bin,Green_bin)
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(signal1,ax, label="Signal1")
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(signal2,ax, label="Signal2")
#%%
fig, (ax0, ax1) = plt.subplots(nrows=2)
ax0=Ananlysis.plot_trace(signal1,ax0, fs=99.384/2, label="signal1 Signal Binned 50Hz",color='g')
ax1=Ananlysis.plot_trace(signal2,ax1, fs=99.384/2, label="signal2 Signal Binned 50Hz", color='r')
fig.tight_layout()
#%%
'''temporal bin'''
bin_window=200
signal1_bin=Ananlysis.get_bin_trace(signal1,bin_window=bin_window)
signal2_bin=Ananlysis.get_bin_trace(signal2,bin_window=bin_window)

fig, (ax0, ax1) = plt.subplots(nrows=2)
ax0=Ananlysis.plot_trace(signal1_bin,ax0, fs=99.384/2, label="signal1 Signal Binned 50Hz",color='r')
ax1=Ananlysis.plot_trace(signal2_bin,ax1, fs=99.384/2, label="signal2 Signal Binned 50Hz", color='g')
fig.tight_layout()

#%%
Red,Green,Signal = getSignalTrace (filename, traceType='TimeDiv',HighFreqRemoval=False,getBinTrace=False)

#%%
# Plot the spectrogram
fig, ax = plt.subplots(figsize=(8, 2))
powerSpectrum, freqenciesFound, time, imageAxis = ax.specgram(signal2_bin, Fs=fs/200,NFFT=1024, detrend='linear',vmin=-130)
ax.set_xlabel('Time (Second)')
ax.set_ylabel('Frequency')
ax.set_ylim([0, 250])
fig.colorbar(imageAxis,ax=ax)

#%%
Ananlysis.PSD_plot (Signal,fs=9938.4/200,method="welch",color='tab:blue',linewidth=1)
fig=Ananlysis.plot_PSD_bands (Signal,fs=9938.4)
#%%
fig=Ananlysis.plot_PSD_bands (trace_binned,fs=9938.4/20)
#%% Low pass filter
'''Get trend and detrend'''
# trace_trend=Ananlysis.butter_filter(trace_clean, btype='low', cutoff=10, fs=9938.4, order=5)
# trace_detrend = Ananlysis.get_detrend(trace_binned)
#%%
'''USE FASTICE method'''
#Red,Green,signal1, signal2 = FreqShift_getICA (trace_clean,fc_g=1000,fc_r=2000,fs=9938.4)

#%%
'''PHOTOMETRY DATA ANALYSIS'''

dpath= "C:/SPAD/SPADData/20220616"
filename=Ananlysis.Set_filename (dpath,csv_filename="1516995_cont-2022-06-16-145825.csv")
Green,Red=Ananlysis.Read_trace (filename,mode="photometry")

fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(Green,ax, fs=130, label="GCamp6 Raw")

Gcamp=Ananlysis.butter_filter(Green, btype='low', cutoff=10, fs=130, order=5)
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(Gcamp,ax, fs=130, label="GCamp6 10Hz lowpass")

fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(Red,ax, fs=130, label="Isospestic Raw", color='m')

Iso=Ananlysis.butter_filter(Red, btype='low', cutoff=10, fs=130, order=5)
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(Iso,ax, fs=130, label="Isospestic 10Hz lowpass", color='m')
#%%
sig=Ananlysis.getSignal_subtract(Red,Green,fs=130)
sig=Ananlysis.butter_filter(sig, btype='low', cutoff=20, fs=130, order=5)
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(sig,ax, fs=130, label="Isospestic")

#%%
Signal=Ananlysis.getSignal_subtract(Red,Green,fs=130)
#%%
signal1,signal2=Ananlysis.getICA (Red,Green)
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(signal1,ax, label="Signal1")
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(signal2,ax, label="Signal2")
#%%
Signal=Ananlysis.butter_filter(Signal, btype='low', cutoff=20, fs=130, order=10)
fig, ax = plt.subplots(figsize=(12, 3))
ax=Ananlysis.plot_trace(Signal,ax, fs=130, label="trace")


### Add fixed time effects and controls for infection levels and national lockdown

# Initial imports
import pandas as pd
import statsmodels.api as sm
import numpy as np
from scipy import stats
import statsmodels.formula.api as smf
import seaborn as sns
import matplotlib.pyplot as plt
import seaborn as sns

#Import household control data
data = pd.read_csv('household_regression_data_weeks.csv')

# Calculate the national lockdown variable for each week
lockdown_weekly = data.groupby('Week')['National_lockdown'].mean().reset_index()

#Import the y_weekly data
y_weekly = pd.read_csv('y_weekly_inf.csv')

#Merge on the national lockdown control and save the file
y_weekly = y_weekly.merge(lockdown_weekly, on='Week')

y_weekly.to_csv('y_weekly_full.csv')

# Fit the OLS regression model
model = smf.ols(formula='Y ~ Treatment + Week + Student_infection_lag + National_lockdown', data=y_weekly).fit()

print(model.summary())

# Plot the residuals
sns.residplot(x=model.predict(), y=y_weekly['Y'], lowess=True, line_kws={'color': 'red'})
plt.title('Residual Plot')
plt.xlabel('Predicted Values')
plt.ylabel('Residuals')
plt.show()

# Plot the real data and model over time
plt.plot(y_weekly['Week'], y_weekly['Y'], label='Real Data')
plt.plot(y_weekly['Week'], model.predict(), label='Model')
plt.axvline(x=10, linestyle='--', color='black')
plt.title('Real Data vs. Model Over Time')
plt.xlabel('Week of Study')
plt.ylabel('% Participation')
plt.ylim((0,1))
plt.legend()
plt.show()


from flask import Flask, request, render_template
import json
import pickle
import nltk
import string
import re
#from nltk.classify import NaiveBayesClassifier

app = Flask(__name__)

#preprocess the text
def preprocess(sentence):
    nltk.download('stopwords')
    nltk.download('punkt')
    def build_bow_features(words):
        return {word:True for word in words}

    sentence = sentence.lower()
    sentence = sentence.replace('\n','')
    useless = nltk.corpus.stopwords.words("english") + list(string.punctuation)
    wordlist = [word for word in nltk.word_tokenize(sentence) if word not in useless]
    stemmed_words = [nltk.stem.SnowballStemmer('english').stem(word) for word in wordlist]
    Bow = (build_bow_features(stemmed_words))
    print(Bow)
    return Bow

#load the trained model and do prediction
def predict (txt):

	prediction = model.classify(txt)
	return prediction

#return the prediction
def submit_txt(txt):
	txt = preprocess(txt)
	status = predict(txt)
	if status==4 :
		return 'Positive'
	if status==0 :
		return 'Negative'
	return 'FAIL'

@app.route('/', methods=['GET', 'POST'])
def index():
	if request.method == 'POST':
		details = request.form
		if details['form_type'] == 'submit_txt':
			return submit_txt(details['txt'])
	return render_template('interface.html')

if __name__ == '__main__':
	model = pickle.load(open('SentimentAnalysisModel2.pkl', 'rb'))
	app.run(host='0.0.0.0')


import os
import glob
import imageio.v2 as imageio
from wand.image import Image
import PySimpleGUI as sg
from moviepy.editor import ImageSequenceClip

# Function to create an MP4 movie from images.
def create_mp4(input_folder, output_path, fps):
    # Get a sorted list of image paths.
    image_paths = sorted(
        [
            os.path.join(input_folder, file)
            for file in os.listdir(input_folder)
            if file.lower().endswith((".png", ".jpg", ".jpeg"))
        ]
    )
    # Read in images.
    images = [imageio.imread(path) for path in image_paths]
    if images:
        # Create and write video file.
        clip = ImageSequenceClip(images, fps=fps)
        clip.write_videofile(output_path, codec="mpeg4")

# Function to create a GIF from images.
def create_gif(input_folder, output_path, duration, loop):
    # Ensure output path ends with .gif.
    if not output_path.lower().endswith(".gif"):
        output_path += ".gif"
    # Get a sorted list of image paths.
    image_paths = sorted(
        [
            os.path.join(input_folder, file)
            for file in os.listdir(input_folder)
            if file.lower().endswith((".png", ".jpg", ".jpeg", ".tiff", ".exr"))
        ]
    )
    # Read in images.
    images = [imageio.imread(path) for path in image_paths]
    if images:
        # Create and write GIF file.
        imageio.mimsave(output_path, images, duration=duration, loop=loop)

# Function to process images with various effects.
def process_images(
    input_folder,
    output_folder,
    dither,
    num_colors,
    pixelate,
    pixelate_factor,
    resize,
    width,
    height,
    rotate,
    angle,
    blur,
    radius,
    mirror,
):
    processing_done = False
    for img_path in glob.glob(os.path.join(input_folder, "*")):
        if img_path.lower().endswith((".png", ".jpg", ".jpeg", ".tiff", ".exr")):
            # Apply requested image processing operations.
            with Image(filename=img_path) as img:
                if dither:
                    img.quantize(number_colors=int(num_colors), dither="riemersma")
                    processing_done = True
                if pixelate:
                    img.resize(
                        int(img.width // pixelate_factor),
                        int(img.height // pixelate_factor),
                    )
                    img.resize(
                        img.width * pixelate_factor, img.height * pixelate_factor
                    )
                    processing_done = True
                if resize:
                    img.resize(width, height)
                    processing_done = True
                if rotate:
                    img.rotate(angle)
                    processing_done = True
                if blur:
                    img.gaussian_blur(radius=radius)
                    processing_done = True
                if mirror:
                    img.flop()
                    processing_done = True
                img.save(
                    filename=os.path.join(output_folder, os.path.basename(img_path))
                )
    return processing_done

# Set up PySimpleGUI.
sg.theme("DarkBlue")

# Define the layout of the GUI.
layout = [
    [
        sg.Text(
            "Process Images bakes input folder images to output folder with selected effects"
        )
    ],
    [
        sg.Text("Input Folder", size=(15, 1)),
        sg.InputText(key="-IN-"),
        sg.FolderBrowse(),
    ],
    [
        sg.Text("Output Folder", size=(15, 1)),
        sg.InputText(key="-OUT-"),
        sg.FolderBrowse(),
    ],
    [
        sg.Checkbox("Dither", key="-DITHER-"),
        sg.InputText(key="-NUM_COLORS-", default_text="256"),
    ],
    [
        sg.Checkbox("Pixelate", key="-PIXELATE-"),
        sg.InputText(key="-PIXELATE_FACTOR-", default_text="1"),
    ],
    [
        sg.Checkbox("Resize", key="-RESIZE-"),
        sg.InputText(key="-WIDTH-", default_text="512"),
        sg.InputText(key="-HEIGHT-", default_text="512"),
    ],
    [
        sg.Checkbox("Rotate", key="-ROTATE-"),
        sg.InputText(key="-ANGLE-", default_text="90"),
    ],
    [sg.Checkbox("Blur", key="-BLUR-"), sg.InputText(key="-RADIUS-", default_text="0")],
    [sg.Checkbox("Mirror", key="-MIRROR-", default=False)],
    [
        sg.Text("MP4 output path"),
        sg.InputText(key="-MP4-"),
        sg.FileSaveAs(file_types=(("MP4 Files", "*.mp4"),)),
    ],
    [sg.Text("MP4 FPS"), sg.InputText(key="-FPS-", default_text="24")],
    [
        sg.Text("GIF output path"),
        sg.InputText(key="-GIF-"),
        sg.FileSaveAs(file_types=(("GIF Files", "*.gif"),)),
    ],
    [sg.Text("GIF delay time in ms"), sg.InputText(key="-DURATION-", default_text="5")],
    [sg.Text("GIF loop count"), sg.InputText(key="-LOOP-", default_text="0")],
    [
        sg.Button("Process Images"),
        sg.Button("Create MP4"),
        sg.Button("Create GIF"),
        sg.Button("Exit"),
    ],
]

# Define icon path.
icon_path = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "data_set_icon_246682.ico"
)
# Create the GUI window.
window = sg.Window("ImageChef", layout, background_color="", icon=icon_path)

# Main event loop.
while True:
    event, values = window.read()
    if event in (None, "Exit"):
        break
    elif event == "Process Images":
        try:
            pixelate_factor = int(values["-PIXELATE_FACTOR-"])
            width = int(values["-WIDTH-"])
            height = int(values["-HEIGHT-"])
            angle = float(values["-ANGLE-"])
            radius = float(values["-RADIUS-"])
            if not values["-IN-"] or not os.path.isdir(values["-IN-"]):
                raise ValueError("Invalid or non-existent input folder")
            if not values["-OUT-"]:
                sg.popup(
                    "No output folder specified, processed images will be saved in the input folder."
                )
        except ValueError as e:
            sg.popup(f"Invalid input: {e}")
            continue

        processing_done = process_images(
            values["-IN-"],
            values["-OUT-"],
            values["-DITHER-"],
            values["-NUM_COLORS-"],
            values["-PIXELATE-"],
            pixelate_factor,
            values["-RESIZE-"],
            width,
            height,
            values["-ROTATE-"],
            angle,
            values["-BLUR-"],
            radius,
            values["-MIRROR-"],
        )
        if processing_done:
            sg.popup("Processing Done")
        else:
            sg.popup("No Processing Done")

    elif event == "Create MP4":
        try:
            fps = int(values["-FPS-"])
            if not values["-IN-"] or not os.path.isdir(values["-IN-"]):
                raise ValueError("Invalid or non-existent input folder")
            if not values["-MP4-"]:
                raise ValueError("Output path for MP4 is empty")
            create_mp4(values["-IN-"], values["-MP4-"], fps)
        except ValueError as e:
            sg.popup(f"Invalid input for MP4 creation: {e}")

    elif event == "Create GIF":
        try:
            duration = float(values["-DURATION-"])
            loop = int(values["-LOOP-"])
            if not values["-OUT-"] or not os.path.isdir(values["-OUT-"]):
                raise ValueError("Invalid or non-existent output folder")
            if not values["-GIF-"]:
                raise ValueError("Output path for GIF is empty")
            create_gif(values["-OUT-"], values["-GIF-"], duration, loop)
        except ValueError as e:
            sg.popup(f"Invalid input for GIF creation: {e}")

window.close()


from django.shortcuts import render

from custom_model_field_app.forms import PersonForm

# Create your views here.

def customview(request):
    form = PersonForm()

    if request.method == 'POST':

        if form.is_valid():
            form.save()

    return render(request,'custom.html',{'form':form})


from langchain.chat_models import ChatVertexAI
from langchain.prompts.chat import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage

chat = ChatVertexAI()

messages = [
    SystemMessage(content="You are a helpful assistant that answer questions."),
    HumanMessage(content="Why is the sky blue?"),
]
print(chat(messages))

template = """
You are a helpful assistant that answer questions.
"""
system_message_prompt = SystemMessagePromptTemplate.from_template(template)
human_template = "{text}"
human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)

chat_prompt = ChatPromptTemplate.from_messages(
    [system_message_prompt, human_message_prompt]
)

# get a chat completion from the formatted messages
resp = chat(chat_prompt.format_prompt(text="why is the sky blue.").to_messages())
print(resp)


from fastexcel import read_excel
from openpyxl import load_workbook
from xlrd import open_workbook

def pyxl_read(test_file_path: str):
    wb = load_workbook(test_file_path, read_only=True, keep_links=False, data_only=True)
    for ws in wb:
        rows = ws.iter_rows()
        rows = ws.values
        for row in rows:
            for value in row:
                value

def xlrd_read(test_file_path: str):
    wb = open_workbook(test_file_path)
    for ws in wb.sheets():
        for idx in range(ws.nrows):
            for value in ws.row_values(idx):
                value

def fastexcel_read(test_file_path: str):
    reader = read_excel(test_file_path)
    for sheet_name in reader.sheet_names:
        sheet = reader.load_sheet_by_name(sheet_name)
        sheet.to_arrow()


import argparse
import pathlib
import itertools
import sys
import urllib

import docker
import tqdm
from compose import config

version = "0.8.0"

def _resolve_name(args, service):
    if args.use_service_image_name_as_filename:
        return urllib.parse.quote(service["image"], safe="")
    return service["name"]

def save(args, client, service, print):
    image = service["image"]
    real_images = [i for i in client.images.list() if image in i.tags]
    if not real_images:
        print("{}: missed (pull, build or specify precisely image name)".format(image))
        sys.exit(1)
    if len(real_images) > 1:
        names = ", ".join(set(itertools.chain.from_iterable(i.tags for i in real_images)))
        print("{}: specify image name more precisely (candidates: {})".format(image, names))
        sys.exit(1)
    path = args.output / "{}.tar".format(_resolve_name(args, service))
    if path.exists() and not args.overwrite:
        print("{} skip ({} already exists)".format(image, path))
        return
    print("{} saving...".format(image))
    args.output.mkdir(parents=True, exist_ok=True)
    with path.open("wb") as f:
        for chunk in real_images[0].save():
            f.write(chunk)

def load(args, client, service, print):
    print("{} loading...".format(service["image"]))
    path = args.input / "{}.tar".format(_resolve_name(args, service))
    with path.open("rb") as f:
        i, *_ = client.images.load(f)
        i.tag(service["image"])

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--version", default=False, action="store_true", help="show version")
    parser.add_argument("--timeout", default=60, type=int, help="docker connection timeout [default: %(default)s]")
    parser.add_argument("--use-service-image-name-as-filename", default=False, action="store_true",
                        help="Support legacy naming behavior")
    parser.add_argument("-f", "--file", default=None, type=pathlib.Path,
                        help="specify an alternate compose file")
    sub_commands = parser.add_subparsers(dest="command")
    sub_commands.required = True
    p = sub_commands.add_parser("save")
    p.set_defaults(function=save)
    p.add_argument("-o", "--output", type=pathlib.Path, default=".",
                   help="output directory [default: %(default)s]")
    p.add_argument("--overwrite", action="store_true", default=False,
                   help="overwrite if exist [default: %(default)s]")
    p = sub_commands.add_parser("load")
    p.set_defaults(function=load)
    p.add_argument("-i", "--input", type=pathlib.Path, default=".",
                   help="input directory [default: %(default)s]")
    return parser.parse_args()

def gen_services(path):
    parent = str(path.parent)
    env = config.environment.Environment.from_env_file(parent)
    details = config.find(parent, [path.name], env)
    resolved = config.load(details)
    for s in resolved.services:
        if "image" not in s:
            raise RuntimeError("Service {!r} have no 'image' field".format(s["name"]))
        yield s

def main():
    args = parse_args()
    if args.version:
        print(version)
        return
    if args.file is None:
        files = ["docker-compose.yml", "docker-compose.yaml"]
    else:
        files = [args.file]
    for file in files:
        path = pathlib.Path(file)
        if not path.exists():
            continue
        path = path.resolve()
        services = list(gen_services(path))
        break
    else:
        raise RuntimeError("Files does not exists {!r}".format(files))
    client = docker.from_env(timeout=args.timeout)
    viewed = set()
    with tqdm.tqdm(total=len(services)) as pbar:
        services.sort(key=lambda s: s["name"])
        for service in services:
            if service["image"] not in viewed:
                args.function(args, client, service, print=pbar.write)
            viewed.add(service["image"])
            pbar.update(1)


import sys
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

# Change default path
# sys.path.append('PROGMOD_')

# Function for formatting file to array of specific variables
# data = txt file
# sh_index = state history index, i.e. controller or environment
# var_index = variable index, i.e. timestamp, joint_angle etc...
def txt_to_variable_array (data, sh_index, var_index):
    sh = data.split("|")[sh_index]
    vars = sh.split("/")[var_index]
    seperated = vars.split("&")

    # Remove blank spaces
    index_to_remove = []
    for i, e in enumerate(seperated):
        if (e == ""):
            index_to_remove.append(i)
    for e in reversed(index_to_remove):
        seperated.pop(e)

    # Convert from strings to float
    in_numbers = [float(numerical_string) for numerical_string in seperated]

    return (in_numbers, seperated, vars)

def plotxy (x, y, y2, xlab, ylab, xunit, yunit):

    # PLOT XY DATA
    plt.plot(x, y, "ro-", color="blue", label="Original")
    plt.plot(x, y2, "ro-", color="red", label="Improved")
    plt.xlabel(xlab)
    plt.ylabel(ylab)

    # SET UNITS
    plt.gca().xaxis.set_major_formatter(ticker.FormatStrFormatter(f'%.1f {xunit}'))
    plt.gca().yaxis.set_major_formatter(ticker.FormatStrFormatter(f'%.1f {yunit}'))

    plt.grid()
    plt.legend()
    plt.show()

# Main
def main():
    # Changing default matplotlib font: https://jonathansoma.com/lede/data-studio/matplotlib/changing-fonts-in-matplotlib/
    matplotlib.rcParams['font.serif'] = "Palatino Linotype" # Change default serif font
    matplotlib.rcParams['font.family'] = "serif" # Set default family to serif

    x_num =          [2, 5, 10, 15, 17, 20, 22, 25, 30, 32]
    y_num_original = [3 ,24, 99, 224, 288, 399, 483, 624, 899, 1023]
    y_num_improved = [4, 13, 28, 43, 49, 58, 64, 73, 88, 94]

    plotxy(x_num, y_num_original, y_num_improved, "No. of processes", "No. of messages", "", "")

if __name__ == "__main__":
    main()


from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
                        print_function, unicode_literals)

from pants.backend.core.tasks.check_exclusives import ExclusivesMapping
from pants.backend.jvm.tasks.jvm_task import JvmTask
from pants.base.exceptions import TaskError
from pants.util.dirutil import safe_mkdtemp, safe_rmtree
from pants_test.task_test_base import TaskTestBase

class DummyJvmTask(JvmTask):
  def execute(self):
    pass

class JvmTaskTest(TaskTestBase):
  """Test some base functionality in JvmTask."""

  @classmethod
  def task_type(cls):
    return DummyJvmTask

  def setUp(self):
    super(JvmTaskTest, self).setUp()
    self.workdir = safe_mkdtemp()

    self.t1 = self.make_target('t1', exclusives={'foo': 'a'})
    self.t2 = self.make_target('t2', exclusives={'foo': 'a'})
    self.t3 = self.make_target('t3', exclusives={'foo': 'b'})
    # Force exclusive propagation on the targets.
    self.t1.get_all_exclusives()
    self.t2.get_all_exclusives()
    self.t3.get_all_exclusives()
    context = self.context(target_roots=[self.t1, self.t2, self.t3])

    # Create the exclusives mapping.
    exclusives_mapping = ExclusivesMapping(context)
    exclusives_mapping.add_conflict('foo', ['a', 'b'])
    exclusives_mapping._populate_target_maps(context.targets())
    context.products.safe_create_data('exclusives_groups', lambda: exclusives_mapping)

    self.task = self.create_task(context, self.workdir)

  def tearDown(self):
    super(JvmTaskTest, self).tearDown()
    safe_rmtree(self.workdir)

  def test_get_base_classpath_for_compatible_targets(self):
    self.task.get_base_classpath_for_compatible_targets([self.t1, self.t2])

  def test_get_base_classpath_for_incompatible_targets(self):
    with self.assertRaises(TaskError):
      self.task.get_base_classpath_for_compatible_targets([self.t1, self.t3])


from ...key import Address
from ...common import Int, concatBytes, _hint
from ...hint import MNFT_COLLECTION_POLICY, MNFT_COLLECTION_REGISTER_FORM, MNFT_MINT_FORM, MNFT_NFT_ID, MNFT_SIGNER, MNFT_SIGNERS

class CollectionRegisterForm:
    def __init__(self, target, symbol, name, royalty, uri, whites):
        assert royalty >= 0 and royalty < 100, 'Invalid royalty; CollectionRegisterForm'
        self.hint = _hint(MNFT_COLLECTION_REGISTER_FORM)
        self.target = Address(target)
        self.symbol = symbol
        self.name = name
        self.royalty = Int(royalty)
        self.uri = uri
        self.whites = []
        for w in whites:
            self.whites.append(Address(w))

    def bytes(self):
        bTarget = self.target.bytes()
        bSymbol = self.symbol.encode()
        bName = self.name.encode()
        bRoyalty = self.royalty.bytes()
        bUri = self.uri.encode()

        _whites = bytearray()
        for w in self.whites:
            _whites += w.bytes()
        bWhites = bytes(_whites)

        return concatBytes(bTarget, bSymbol, bName, bRoyalty, bUri, bWhites)

    def dict(self):
        form = {}
        form['_hint'] = self.hint.hint
        form['target'] = self.target.address
        form['symbol'] = self.symbol
        form['name'] = self.name
        form['royalty'] = self.royalty.value
        form['uri'] = self.uri

        whites = []
        for w in self.whites:
            whites.append(w.address)
        form['whites'] = whites

        return form

class CollectionPolicy:
    def __init__(self, name, royalty, uri, whites):
        assert royalty >= 0 and royalty < 100, 'Invalid royalty; CollectionPolicy'
        self.hint = _hint(MNFT_COLLECTION_POLICY)
        self.name = name
        self.royalty = Int(royalty)
        self.uri = uri
        self.whites = []
        for w in whites:
            self.whites.append(Address(w))

    def bytes(self):
        bName = self.name.encode()
        bRoyalty = self.royalty.bytes()
        bUri = self.uri.encode()

        _whites = bytearray()
        for w in self.whites:
            _whites += w.bytes()
        bWhites = bytes(_whites)

        return concatBytes(bName, bRoyalty, bUri, bWhites)

    def dict(self):
        policy = {}
        policy['_hint'] = self.hint.hint
        policy['name'] = self.name
        policy['royalty'] = self.royalty.value
        policy['uri'] = self.uri

        whites = []
        for w in self.whites:
            whites.append(w.address)
        policy['whites'] = whites

        return policy

class NFTSigner:
    def __init__(self, account, share, signed):
        assert share >= 0 and share <= 100, 'Invalid share; NFTSigner'
        self.hint = _hint(MNFT_SIGNER)
        self.account = Address(account)
        self.share = Int(share)
        self.signed = signed

    def bytes(self):
        bAccount = self.account.bytes()
        bShare = self.share.bytes()
        bSigned = bytes([0])
        if self.signed:
            bSigned = bytes([1])
        return concatBytes(bAccount, bShare, bSigned)

    def dict(self):
        signer = {}
        signer['_hint'] = self.hint.hint
        signer['account'] = self.account.address
        signer['share'] = self.share.value
        signer['signed'] = self.signed
        return signer

class NFTSigners:
    def __init__(self, total, signers):
        assert total >= 0 and total <= 100, 'Invalid total share; NFTSigners'
        self.hint = _hint(MNFT_SIGNERS)
        self.total = Int(total)
        self.signers = signers

    def bytes(self):
        bTotal = self.total.bytes()
        _signers = bytearray()
        for s in self.signers:
            _signers += s.bytes()
        bSigners = _signers

        return concatBytes(bTotal, bSigners)

    def dict(self):
        signers = {}
        signers['_hint'] = self.hint.hint
        signers['total'] = self.total.value
        _signers = []
        for s in self.signers:
            _signers.append(s.dict())
        signers['signers'] = _signers

        return signers

class MintForm:
    def __init__(self, hash, uri, creators, copyrighters):
        self.hint = _hint(MNFT_MINT_FORM)
        self.hash = hash
        self.uri = uri
        self.creators = creators
        self.copyrighters = copyrighters

    def bytes(self):
        bHash = self.hash.encode()
        bUri = self.uri.encode()
        bCreators = self.creators.bytes()
        bCopyrighters = self.copyrighters.bytes()

        return concatBytes(bHash, bUri, bCreators, bCopyrighters)

    def dict(self):
        form = {}
        form['_hint'] = self.hint.hint
        form['hash'] = self.hash
        form['uri'] = self.uri
        form['creators'] = self.creators.dict()
        form['copyrighters'] = self.copyrighters.dict()
        return form

class NFTID:
    def __init__(self, collection, idx):
        assert idx.value > 0, 'idx must be over zero; NFTID'
        self.hint = _hint(MNFT_NFT_ID)
        self.collection = collection
        self.idx = idx

    def bytes(self):
        bCollection = self.collection.encode()
        bIdx = self.idx.bytes()
        return concatBytes(bCollection, bIdx)

    def dict(self):
        id = {}
        id['_hint'] = self.hint.hint
        id['collection'] = self.collection
        id['idx'] = self.idx.value
        return id


from PyQt5 import QtCore
from PyQt5.QtCore import QObject, QThreadPool, pyqtSignal
from PyQt5.QtWidgets import QWidget, QScrollArea

from cvstudio.util import GUIUtilities
from cvstudio.view.widgets import ImageButton
from cvstudio.view.widgets.loading_dialog import QLoadingDialog
from cvstudio.view.widgets.response_grid import ResponseGridLayout, GridCard
from cvstudio.view.wizard import ModelWizard

class ModelsGridWidget(QWidget, QObject):
    new_item_action = pyqtSignal()

    def __init__(self, parent=None):
        super(ModelsGridWidget, self).__init__(parent)
        self.grid_layout = ResponseGridLayout()
        self.grid_layout.setAlignment(QtCore.Qt.AlignTop)
        self.grid_layout.cols = 8
        self.setLayout(self.grid_layout)
        self._entries = None

    def build_new_button(self):
        new_item_widget: GridCard = GridCard(with_actions=False, with_title=False)
        btn_new_item = ImageButton(GUIUtilities.get_icon("new_folder.png"))
        btn_new_item.clicked.connect(lambda: self.new_item_action.emit())
        new_item_widget.body = btn_new_item
        return new_item_widget

    def bind(self):
        cards_list = []
        new_item_button = self.build_new_button()
        cards_list.append(new_item_button)
        self.grid_layout.widgets = cards_list
        super(ModelsGridWidget, self).update()

class ModelsTabWidget(QScrollArea):
    def __init__(self, parent=None):
        super(ModelsTabWidget, self).__init__(parent)
        self.setCursor(QtCore.Qt.PointingHandCursor)
        self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)
        self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
        self.data_grid = ModelsGridWidget()
        self.data_grid.new_item_action.connect(self.data_grid_new_item_action_slot)
        self.setWidget(self.data_grid)
        self.setWidgetResizable(True)
        self._thread_pool = QThreadPool()
        self._loading_dialog = QLoadingDialog()
        self.load()

    def data_grid_new_item_action_slot(self):
        new_model_wizard = ModelWizard()
        new_model_wizard.exec_()

    def load(self):
        self.data_grid.bind()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 24 06:56:45 2018

@author: Javier Alejandro Acevedo Barroso
"""

import numpy as np
import matplotlib.pyplot as plt

x_obs = np.array([-2.0,1.3,0.4,5.0,0.1, -4.7, 3.0, -3.5,-1.1])
y_obs = np.array([ -1.931,   2.38,   1.88,  -24.22,   3.31, -21.9,  -5.18, -12.23,   0.822])
sigma_y_obs = ([ 2.63,  6.23, -1.461, 1.376, -4.72,  1.313, -4.886, -1.091,  0.8054])
plt.errorbar(x_obs, y_obs, yerr=sigma_y_obs, fmt='o')

def model(x,a,b,c):
    return a*x*x  + b * x + c

def loglikelihood(x_obs, y_obs, sigma_y_obs, a, b, c):
    d = y_obs -  model(x_obs, a,b,c)
    d = d/sigma_y_obs
    d = -0.5 * np.sum(d**2)
    return d

def logprior(a, b, c):
#    p = -np.inf
#    if a < 2 and a >-2 and b >-10 and b<10 and c > -10 and c < 10:
#        p = 0.0
#    return p
    return 0

N= 50000
lista_a = [np.random.random()]
lista_b = [np.random.random()]
lista_c = [np.random.random()]
logposterior = [loglikelihood(x_obs, y_obs, sigma_y_obs, lista_a[0], lista_b[0], lista_c[0]) + logprior(lista_a[0], lista_b[0], lista_c[0])]

sigma_delta_a = 0.2
sigma_delta_b = 1
sigma_delta_c = 1.0

for i in range(1,N):
    propuesta_a  = lista_a[i-1] + np.random.normal(loc=0.0, scale=sigma_delta_a)
    propuesta_b  = lista_b[i-1] + np.random.normal(loc=0.0, scale=sigma_delta_b)
    propuesta_c  = lista_b[i-1] + np.random.normal(loc=0.0, scale=sigma_delta_c)

    logposterior_viejo = loglikelihood(x_obs, y_obs, sigma_y_obs, lista_a[i-1], lista_b[i-1], lista_c[i-1]) + logprior(lista_a[i-1], lista_b[i-1], lista_c[i-1])
    logposterior_nuevo = loglikelihood(x_obs, y_obs, sigma_y_obs, propuesta_a, propuesta_b, propuesta_c) + logprior(propuesta_a, propuesta_b,propuesta_c)

    r = min(1,np.exp(logposterior_nuevo-logposterior_viejo))
    alpha = np.random.random()
    if(alpha<r):
        lista_a.append(propuesta_a)
        lista_b.append(propuesta_b)
        lista_c.append(propuesta_c)
        logposterior.append(logposterior_nuevo)
    else:
        lista_a.append(lista_a[i-1])
        lista_b.append(lista_b[i-1])
        lista_c.append(lista_c[i-1])
        logposterior.append(logposterior_viejo)
lista_a = np.array(lista_a)
lista_b = np.array(lista_b)
lista_c = np.array(lista_c)
logposterior = np.array(logposterior)

realx = np.linspace(-5,5,100)
#rta = (lista_a.argmax(),lista_b.argmax(),lista_c.argmax())
rta = (lista_a.mean(),lista_b.mean(),lista_c.mean())
plt.plot(realx, model(realx,rta[0],rta[1],rta[2]))
plt.title("a = %.3f  b = %.3f  c = %.3f" % (rta))

h2 = plt.figure()

#plt.plot(lista_a[100:], label='pendiente')
#plt.plot(lista_b[100:], label='intercepto')
#plt.plot(lista_c[100:], label='c')
#plt.plot(logposterior[100:], label='loglikelihood')
#plt.legend()

h2 = plt.figure()
plt.plot(lista_a, lista_b, alpha=0.5)
plt.scatter(lista_a, lista_b, alpha=0.4, c=np.exp(logposterior))
plt.colorbar()

h2 = plt.figure()
plt.plot(lista_b, lista_c, alpha=0.5)
plt.scatter(lista_a, lista_b, alpha=0.4, c=np.exp(logposterior))
plt.colorbar()

h2 = plt.figure()
plt.plot(lista_a, lista_c, alpha=0.5)
plt.scatter(lista_a, lista_b, alpha=0.4, c=np.exp(logposterior))
plt.colorbar()


import numpy as np
from utils import plot_output

# Defining parameters
N = 500
L = 5  # Topological charge number
A3 = np.zeros((N, N), dtype='complex_')
# Constructing SPP
x = np.array([i for i in range(N)])
y = np.array([i for i in range(N)])

X, Y = np.meshgrid(x, y)
theta = np.arctan2((X - N/2), (Y - N/2))
r = np.sqrt((X - N/2) * (X - N/2) + (Y - N/2) * (Y - N/2))
A1 = L * (theta + np.pi)
A2 = np.fmod(A1, 2*np.pi)

for p in range(N):
    for q in range(N):
        if np.fmod(A2[p, q], 2*np.pi) <= np.pi:
            A3[p, q] = np.exp(1j * np.pi)
        else:
            A3[p, q] = np.exp(1j * 0)
A3[r > 30] = 0

plot_output(A3, N, angle=True, spiral=True)


# https://pypi.org/project/RPi.bme280/
import smbus2
import bme280

import syslog
import threading
def logmsg(level, msg):
    syslog.syslog(level, 'station: {}: {}'.format(threading.currentThread().getName(), msg))

def logdbg(msg):
    logmsg(syslog.LOG_DEBUG, msg)

def loginf(msg):
    logmsg(syslog.LOG_INFO, msg)

def logerr(msg):
    logmsg(syslog.LOG_ERR, msg)

port = 1
address = 0x76
bus = smbus2.SMBus(port)

calibration_params = bme280.load_calibration_params(bus, address)

import numpy as np
import time as tm
import os
#with open('station.log', 'w') as f:
#	pass
target_t_f = 85.
t_safe = 90.
from importlib import reload
cadence = 5 # seconds
last_time_s = None
last_error = None
while True:
	data = bme280.sample(bus, address, calibration_params)

	t_f = data.temperature * 9. / 5 + 32.
	with open('status', 'r') as f:
		status = f.readline()
	output = '{:30} {:10s} {:5.2f} {:8.2f} {:5.2f}'.format(str(data.timestamp).replace(' ','_'), status, t_f, data.pressure, data.humidity)
	with open('station.log', 'a') as f:
		#f.write('{} error={:.3f} dt={:6.3f}s p={:.3f} i={:.3f} d={:.3f} pid={:.3f}'.format(output, error, dt, p, i, d, pid))
		f.write('{}\n'.format(output))
	time_s = tm.mktime(data.timestamp.timetuple())

	if t_f > t_safe:
		print('temperature {:.2f} > safe temperature {:.2f}; safe mode'.format(t_f, t_safe))
		os.system('./safe')
	#elif t_f < target_t_f:
	#	os.system('./on')
	#else:
	#	os.system('./off')
	else:
		error = t_f - target_t_f
		kp = 1
		ki = kp / 300 # 5-minute integration time
		kd = kp * 300 # 5-minute derivative time
		p = kp * error
		if last_time_s:
			dt = time_s - last_time_s
			i += ki * 0.5 * (error + last_error) * dt
			d = kd * (error - last_error) / dt
		else:
			dt = 0
			i = 0
			d = 0
		# now instead of comparing t_f to target_t_f, compare p+i+d to zero
		last_time_s = time_s
		last_error = error

		pid = p + i + d
		full_output = '{} error={:.3f} dt={:6.3f}s p={:.3f} i={:.3f} d={:.3f} pid={:.3f}'.format(output, error, dt, p, i, d, pid)
		print(full_output)
		loginf(full_output)
		if pid > 0:
			try:
				os.system('./off')
			except Exception as e:
				logerr(str(e.args[0]))
				with open('station.e', 'a') as ferr:
					ferr.write('{} {}\n'.format(str(e), full_output))
		elif pid < 0:
			try:
				os.system('./on')
			except Exception as e:
				logerr(str(e.args[0]))
				with open('station.e', 'a') as ferr:
					ferr.write('{} {}\n'.format(str(e), full_output))
		else:
			logerr('error {} not a number'.format(pid))
			raise ValueError('error not a number')

	tm.sleep(cadence)


import abc
import numpy as np
from nn.utils.label_mapper import LabelMapper
from datetime import datetime

class LearningAlgorithmTypes(object):
    SGD = "stochastic gradient descent"

class LearningAlgorithmFactory(object):
    @staticmethod
    def create_learning_algorithm_from_type(learning_algorithm_type):
        if learning_algorithm_type == LearningAlgorithmTypes.SGD:
            return SGD()
        else:
            raise NotImplementedError("Requested learning algorithm type not yet implemented")

class AbstractLearningAlgorithm(object):
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def learn(self, *args, **kwargs):
        raise NotImplementedError()

class SGD(AbstractLearningAlgorithm):
    def __init__(self):
        super(SGD, self).__init__()

    def learn(self, network, training_data_set, number_of_epochs, learning_rate, size_of_batch, **kwargs):
        for epoch in range(number_of_epochs):
            print("Epoch: " + str(epoch) + " Start time: " + str(datetime.now()))
            np.random.shuffle(training_data_set.data_instances)

            for batch in np.array_split(training_data_set.data_instances,
                                        len(training_data_set.data_instances) / size_of_batch):
                self.__update_weights_and_bias(network, batch, learning_rate)

    def __update_weights_and_bias(self, network, batch, learning_rate):
        number_of_training_instances = len(batch)
        updated_biases = map(lambda layer_biases: np.zeros(layer_biases.shape), network.biases)
        updated_weights = map(lambda layer_weights: np.zeros(layer_weights.shape), network.weights)

        for data_instance in batch:
            # Computing the partial derivatives of the function cost w.r.t. each weight and bias. These partial
            # derivatives are the gradient's components.
            delta_biases, delta_weights = self.__back_propagate(network, data_instance.features_values_vector,
                                                                data_instance.label)

            # Accumulating the delta of weights and biases for each training sample of the batch in order to adjust
            # the network's weights and biases in the opposite direction of the gradient.
            updated_biases = [new_bias + delta for new_bias, delta in zip(updated_biases, delta_biases)]
            updated_weights = [new_weight + delta for new_weight, delta in zip(updated_weights, delta_weights)]

        # Updating the network's weights and biases in the opposite direction of the cost function's gradient
        network.weights = [current_weight - (learning_rate / number_of_training_instances) * new_weight
                           for current_weight, new_weight in zip(network.weights, updated_weights)]
        network.biases = [current_bias - (learning_rate / number_of_training_instances) * new_bias
                          for current_bias, new_bias in zip(network.biases, updated_biases)]

    def __back_propagate(self, network, output_vector, expected_output_label):
        last_layer = -1
        updated_biases = map(lambda layer_biases: np.zeros(layer_biases.shape), network.biases)
        updated_weights = map(lambda layer_weights: np.zeros(layer_weights.shape), network.weights)

        output_vectors_by_layer = [output_vector.reshape(1, len(output_vector))]
        input_vectors_by_layer = []

        for bias, weights in zip(network.biases, network.weights):
            next_layer_input = np.dot(output_vector, weights) + bias.T
            input_vectors_by_layer.append(next_layer_input)
            output_vector = network.neuron.compute(next_layer_input)
            output_vectors_by_layer.append(output_vector)

        delta = network.cost_computer.compute_cost_derivative(output_vector=output_vectors_by_layer[last_layer],
                                                              expected_output_vector=LabelMapper().map_label_to_vector(
                                                                      expected_output_label)) * \
                network.neuron.compute_derivative(input_vectors_by_layer[last_layer])
        updated_biases[last_layer] = delta.T
        updated_weights[last_layer] = np.dot(output_vectors_by_layer[last_layer - 1].T, delta)

        for layer_index in xrange(2, network.number_of_layers):
            z = input_vectors_by_layer[-layer_index]
            sp = network.neuron.compute_derivative(z)
            delta = np.dot(delta, network.weights[-layer_index + 1].T) * sp
            updated_biases[-layer_index] = delta.T
            updated_weights[-layer_index] = np.dot(output_vectors_by_layer[-layer_index - 1].T, delta)

        return updated_biases, updated_weights


'''
    Fig1. Cascade Matrix heatmap
'''

import numpy as np
import matplotlib.pyplot as plt
# from numba import jit
import matplotlib.colors as colors

# @jit(nopython = True)
def random_cascade_matrix(mu_L, mu_U, sigma_L, sigma_U, gamma, N = 250):
    J = np.zeros((N, N))

    for i in range(N):
        for j in range(i):
            z1 = np.random.normal(0, 1)
            z2 = np.random.normal(0, 1)

            J[i, j] = mu_L/N + sigma_L*z1/np.sqrt(N)
            J[j, i] = mu_U/N + sigma_U*(gamma*z1 + np.sqrt(1 - gamma**2)*z2)/np.sqrt(N)

    return J
fig, axs = plt.subplots(1, 3, figsize = (10, 3))
plt.rc('text', usetex=True)
plt.rcParams.update({'font.size': 12})
# for display
def block_matrix(mu, sizes):
    N = np.sum(sizes)
    c = np.hstack((np.array([0]), np.cumsum(sizes)[:-1]))

    J = np.zeros((N, N))

    for i in range(np.size(sizes)):
        for j in range(np.size(sizes)):
            for u in range(sizes[i]):
                for v in range(sizes[j]):
                    J[c[i] + u, c[j] + v] = mu[i, j]

    return J

def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):
    new_cmap = colors.LinearSegmentedColormap.from_list(
        'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),
        cmap(np.linspace(minval, maxval, n)))
    return new_cmap

arr = np.linspace(0, 50, 100).reshape((10, 10))

cmap = plt.get_cmap('viridis')
new_cmap = truncate_colormap(cmap, 0.12, 0.98)

mu1 = np.random.uniform(0, 1, 16).reshape((4, 4))
sizes1 = np.array([5, 4, 7, 3])

mu_cascade = np.array([[1, 2, 2, 2], [0, 1, 2, 2], [0, 0, 1, 2], [0, 0, 0, 1]])
sizes_cascade = np.array([7, 5, 6, 4])
J_cascade = block_matrix(mu_cascade, sizes_cascade)

J1 = block_matrix(mu1, sizes1)
axs[0].matshow(J1, cmap = "viridis")
axs[1].matshow(J_cascade, cmap = new_cmap)
axs[2].matshow(random_cascade_matrix(-1, 1, 0.0, 0.0, 0.0, 500), cmap = new_cmap)

# print(random_cascade_matrix)
# for i in range(3):
    # Turn off tick labels
axs[0].set_xticklabels([])
axs[0].set_xticks([])

axs[1].set_xticklabels([])
axs[1].set_xticks([])

axs[2].set_xticklabels([])
axs[2].set_xticks([])

axs[0].get_yaxis().set_visible(False)
axs[1].get_yaxis().set_visible(False)
axs[2].get_yaxis().set_visible(False)

axs[0].set_xlabel("Block Structured")
axs[1].set_xlabel("Cascade Model, Finite B")
axs[2].set_xlabel("Cascade Model, infinite B")
plt.tight_layout()

# panel labels
axs[0].annotate('(a)', xy=(0.05, 0.05), xycoords='axes fraction', zorder = 10, ma = 'center', bbox=dict(facecolor='white', alpha=0.6, boxstyle='round'))
axs[1].annotate('(b)', xy=(0.05, 0.05), xycoords='axes fraction', zorder = 10, ma = 'center', bbox=dict(facecolor='white', alpha=0.6, boxstyle='round'))
axs[2].annotate('(c)', xy=(0.05, 0.05), xycoords='axes fraction', zorder = 10, ma = 'center', bbox=dict(facecolor='white', alpha=0.6, boxstyle='round'))
fig.tight_layout()
plt.savefig("Interaction Matrix Cartoon.pdf")


import numpy as np
from matplotlib.pyplot import *
from scipy import interpolate

dat=np.loadtxt("/home/davidvartanyan/presupernova.dat")
rad=dat[:,2]
rho1=dat[:,4]
i=0
while rad[i]/10**9 < 1:
    i+=1
xlim([rad[0],rad[i]])
#loglog(rad[0:i],rho1[0:i],'k')
npoints=1000
radmin=rad[0]
radmax=10**9
radius=np.linspace(0.1,
                   radmax,npoints)
dr=radius[1]-radius[0]

tck = interpolate.splrep(rad,rho1)
rho2= interpolate.splev(radius,tck,der=0)

#xlabel('radius [$cm$]')
#ylabel('Density [$g/cm^3$]')
#show()

ggrav = 6.67e-8

#######################################
# function definitions
def tov_RHS(x,rho,z):

    # RHS function

    rhs = np.zeros(2)
    rhs[0] = z
    rhs[1] = 4*np.pi*ggrav*rho- 2*z/x

    return rhs

def tov_integrate_FE(rad,dr,rho,z,phi):

    # Forward-Euler Integrator

    new = np.zeros(2)
    old = np.zeros(2)
    old[0] = phi #dphi/dr-> phi
    old[1] = z #dz/dr -> z

    # forward Euler integrator
    new = old + dr*tov_RHS(rad, rho, z)

    # assign outputs
    phinew = new[0]
    znew = new[1]

    return (phinew,znew)

#######################################
# set up variables
z1f   = np.zeros(npoints)
phif  = np.zeros(npoints)
# set up boundary values
z1f[0] = 0.0
phif[0] = 0.0

for n in range(npoints-1):

    (phif[n+1],z1f[n+1]) = tov_integrate_FE(radius[n],
                                              dr,rho2[n],z1f[n],phif[n])
dm=4*np.pi*radius**2*rho2*dr
M=np.sum(dm)

phiBC2=-ggrav*M/radius[npoints-1]

phiana=2./3*np.pi*ggrav*rho2*(radius**2-3*((10**9.)**2))
phifin=phif+phiBC2-phif[npoints-1]

#p1,=loglog(radius,-phifin)
#p2,=loglog(radius,-phiana)
xlabel('radius[cm]')
#ylabel('Phi')
#legend([p1,p2], ['Numerical Potential', 'Analytical Potential'])
ylabel('Error')
loglog(radius,np.abs((phiana-phifin)/phiana))
show()

#radmax2=np.asarray([10**9]*len(radius))
#origin=np.asarray([4.2118*10**20]*len(radius))
#plot(radius,phif+origin)

#print .67*np.pi*ggrav*rho2*(radius**2-3*radmax2**2)
#print  -0.67*np.pi*ggrav*rho2*(radius**2-3*radmax2**2)


import logging

from aiogram import Dispatcher, types
from aiogram.dispatcher import FSMContext
import aiogram.utils.markdown as fmt
from aiogram.types.message import ContentType

from .. import userchoice

def check_none_name(name):
    new_name = ''
    if name is not None:
        new_name = name
    return new_name

async def starting_message(message: types.Message, state: FSMContext):
    """
    Первая команда, она проверяет существование пользователя и заполняет его данные

    :param message: сообщение
    :param state: состояние
    """
    await state.finish()

    first_name = check_none_name(message.from_user.first_name)
    last_name = check_none_name(message.from_user.last_name)

    user = userchoice.UserChoice(message.from_user.id, first_name + ' ' + last_name)
    user.check_name()

    logging.info(f'Пользователь {message.from_user.first_name} {message.from_user.last_name} залогинился')

    await message.answer("Привет! Этот бот сравнивает позволяет сравнить фамилии по "
                         "рейтингу Эло. "
                         "Для перечня команд набери /help", parse_mode=types.ParseMode.HTML)

async def helping_message(message: types.Message):
    """
    Сообщение выводит существующие команды

    :param message: сообщение
    """
    await message.answer(fmt.text("Я знаю следующие команды:", "/rate - выбрать более смешного из пары",
                                  "/rating - показать список лидеров", sep='\n'))

async def wrong_command_message(message: types.Message):
    """
    Сообщение реагирует на неправильные команды

    :param message: сообщение
    """

    logging.info(f'Пользователь {message.from_user.first_name} {message.from_user.last_name} пишет {message.text}')

    await message.answer("Вы ввели неверную команду. Для того, чтобы узнать, "
                         "какие команды можно использовать, наберите /help")

def register_handlers_common(dp: Dispatcher):
    """
    Регистрация основных сообщений в диспетчере

    :param dp: диспетчер
    """
    dp.register_message_handler(starting_message, commands="start", state="*")
    dp.register_message_handler(helping_message, commands="help")
    dp.register_message_handler(wrong_command_message, content_types=ContentType.ANY)


import os
import os.path
import shutil
import tarfile
import hashlib
import argparse
import fnmatch
import sys

STR_EMPTY = ''
STR_SLASH = '/'
STR_POINT = '.'
STR_TAB = '\t'
STR_EOL = '\n'
STR_CAT_EXT = '.cat'
STR_TAR_EXT = '.tar'
STR_GZ_EXT = '.tar.gz'
STR_BZ2_EXT = '.tar.bz2'

STR_DIR_LIST = 'DIR_LIST'
STR_DIR = 'DIR'
STR_DIR_END = 'DIR_END'
STR_FILE = 'FILE'
STR_FILE_END = 'FILE_END'
STR_DIR_LIST_END = 'DIR_LIST_END'

STR_HASH_LIST = 'HASH_LIST'
STR_HASH = 'HASH'
STR_HASH_LIST_END = 'HASH_LIST_END'

def calc_hash(path):  # IOError
    h = hashlib.sha256()

    with open(path, 'rb') as f:  # IOError
        block = f.read(h.block_size)
        while block:
            h.update(block)
            block = f.read(h.block_size)

    return h.hexdigest()

class CatalogFormatError(Exception):
    pass

class HashNameError(Exception):
    pass

class FileInfo():
    def __init__(self, is_dir):
        self.marked = False  # for 'include'
        self.isDir = is_dir
        self.hash = STR_EMPTY
        self.size = -1

def hash_name(info):  # HashNameError
    if info.isDir or (info.hash == STR_EMPTY) or (info.size == -1):
        raise HashNameError()
    return info.hash + '.' + str(info.size)

class FileList():  # OSError, IOError, CatalogFormatError
    def __init__(self):
        self.dict = {}

    def _get_dir_list(self, root_dir, rel_dir=STR_EMPTY):  # OSError
        if rel_dir == STR_EMPTY:
            self.dict.clear()
        current_dir = root_dir + rel_dir
        current_dir_list = os.listdir(current_dir)  # OSError
        for f in current_dir_list:
            full_path = current_dir + STR_SLASH + f
            rel_path = rel_dir + STR_SLASH + f
            if os.path.isdir(full_path):  # OSError
                path_info = FileInfo(True)
                path_info.mtime = int(os.path.getmtime(full_path))  # OSError
                self.dict[rel_path] = path_info
                self._get_dir_list(root_dir, rel_path)
            elif os.path.isfile(full_path):  # OSError
                path_info = FileInfo(False)
                # read mtime, size and hash directly before file checking / archiving
                self.dict[rel_path] = path_info

    def read_dir_list(self, source_path):
        try:
            self._get_dir_list(source_path)
        except IOError as e:
            print('ERROR: Can not read: ' + e.filename)
            return

    def _unmark_all(self):
        for key in self.dict:
            self.dict[key].marked = False

    # include only matched files/folders
    # use for "find"
    def include(self, pattern_list):
        if (pattern_list is not None) and (len(pattern_list) > 0):
            # unmark all records
            self._unmark_all()
            # mark included
            for pattern in pattern_list:
                for key in self.dict:
                    if fnmatch.fnmatch(key, pattern):
                        self.dict[key].marked = True
            # remove not marked (not included)
            key_list = list(self.dict.keys())
            for key in key_list:
                if not self.dict[key].marked:
                    del self.dict[key]

    # include not only matched files/folders but also all parent folders for matched files/folders
    # use for "create" and "restore"
    def include_hierarchy(self, pattern_list):
        if (pattern_list is not None) and (len(pattern_list) > 0):
            # unmark all records
            self._unmark_all()
            # mark included
            key_list = list(self.dict.keys())
            for pattern in pattern_list:
                for key in key_list:
                    if fnmatch.fnmatch(key, pattern):
                        self.dict[key].marked = True
                        # mark folders with marked files/folders
                        d = os.path.dirname(key)
                        while d != STR_SLASH:
                            self.dict[d].marked = True
                            d = os.path.dirname(d)
            # remove not marked (not included)
            key_list = list(self.dict.keys())
            for key in key_list:
                if not self.dict[key].marked:
                    del self.dict[key]

    # check and if not exist all parent folders for files/folders in list
    def fix_hierarchy(self):
        key_list = list(self.dict.keys())
        for key in key_list:
            d = os.path.dirname(key)
            while d != STR_SLASH:
                if d not in key_list:
                    path_info = FileInfo(False)
                    path_info.marked = False  # for 'include'
                    path_info.isDir = True
                    path_info.mtime = self.dict[key].mtime
                    self.dict[d] = path_info
                d = os.path.dirname(d)

    def exclude(self, pattern_list):
        if (pattern_list is not None) and (len(pattern_list) > 0):
            for pattern in pattern_list:
                key_list = list(self.dict.keys())
                for key in key_list:
                    if fnmatch.fnmatch(key, pattern):
                        del self.dict[key]

    def save(self, file_object):  # IOError
        # file_object = open('file.name', mode='w', encoding='utf-8')
        file_object.write(STR_DIR_LIST + STR_EOL)
        key_list = list(self.dict.keys())
        key_list.sort()
        for key in key_list:
            if self.dict[key].isDir:
                file_object.write(STR_DIR + STR_EOL)
                file_object.write(key + STR_EOL)
                file_object.write(str(self.dict[key].mtime) + STR_EOL)
                file_object.write(STR_DIR_END + STR_EOL)
            else:
                file_object.write(STR_FILE + STR_EOL)
                file_object.write(key + STR_EOL)
                file_object.write(str(self.dict[key].mtime) + STR_EOL)
                file_object.write(str(self.dict[key].size) + STR_EOL)
                file_object.write(self.dict[key].hash + STR_EOL)
                file_object.write(STR_FILE_END + STR_EOL)
        file_object.write(STR_DIR_LIST_END + STR_EOL)

    def load(self, file_object):  # IOError, CatalogFormatError
        # file_object = open('file.name', mode='r', encoding='utf-8')
        wait_list = 0
        wait_dir_file = 1
        wait_path = 2
        wait_mtime = 3
        wait_size = 4
        wait_hash = 5
        wait_dir_end = 6
        wait_file_end = 7

        self.dict.clear()
        file_object.seek(0, os.SEEK_SET)

        state = wait_list
        info_is_dir = False
        info_path = STR_EMPTY
        info_mtime = -1
        info_size = -1
        info_hash = STR_EMPTY
        for s in file_object:
            line = s.strip()
            if (state == wait_list) and (line == STR_DIR_LIST):
                state = wait_dir_file

            elif ((state == wait_dir_file) and
                  ((line == STR_DIR) or (line == STR_FILE) or (line == STR_DIR_LIST_END))):
                if line == STR_DIR:
                    info_is_dir = True
                    state = wait_path
                elif line == STR_FILE:
                    info_is_dir = False
                    state = wait_path
                elif line == STR_DIR_LIST_END:
                    return

            elif state == wait_path:
                info_path = line
                state = wait_mtime

            elif state == wait_mtime:
                info_mtime = int(line)
                if info_is_dir:
                    state = wait_dir_end
                else:
                    state = wait_size

            elif state == wait_size:
                info_size = int(line)
                state = wait_hash

            elif state == wait_hash:
                info_hash = line
                state = wait_file_end

            elif (state == wait_dir_end) and (line == STR_DIR_END):
                self.dict[info_path] = FileInfo(True)
                self.dict[info_path].mtime = info_mtime
                info_is_dir = False
                state = wait_dir_file

            elif (state == wait_file_end) and (line == STR_FILE_END):
                self.dict[info_path] = FileInfo(False)
                self.dict[info_path].mtime = info_mtime
                self.dict[info_path].size = info_size
                self.dict[info_path].hash = info_hash
                state = wait_dir_file

            else:
                raise CatalogFormatError()  # CatalogFormatError

    def load_file(self, file_name):
        try:
            file_object = open(file_name, mode='r', encoding='utf-8')
            try:
                self.load(file_object)
            except IOError:
                print('ERROR: Can not read reference catalogue file!')
                return
            except CatalogFormatError:
                print('ERROR: Reference catalogue is damaged!')
                return
            finally:
                file_object.close()
        except IOError:
            print('ERROR: Can not open reference catalogue file!')

# key = hash + u'.' + unicode(size)
# value = arch name
# FileList.dict[key].hashName
class HashList():  # IOError, CatalogFormatError
    def __init__(self):
        self.dict = {}

    def save(self, file_object):  # IOError
        # file_object = open('file.name', mode='w', encoding='utf-8')
        file_object.write(STR_HASH_LIST + STR_EOL)
        key_list = list(self.dict.keys())
        key_list.sort()
        for key in key_list:
            file_object.write(STR_HASH + STR_TAB + key + STR_TAB + self.dict[key] + STR_EOL)
        file_object.write(STR_HASH_LIST_END + STR_EOL)

    def load(self, file_object):  # IOError, CatalogFormatError
        # file_object = open('file.name', mode='r', encoding='utf-8')
        wait_list = 0
        wait_hash = 1
        self.dict.clear()
        file_object.seek(0, os.SEEK_SET)

        state = wait_list
        for s in file_object:
            line = s.strip()
            if (state == wait_list) and (line == STR_HASH_LIST):
                state = wait_hash
            elif state == wait_hash:
                if line == STR_HASH_LIST_END:
                    return
                else:
                    lst = line.split(STR_TAB)
                    if (len(lst) == 3) and (lst[0] == STR_HASH):
                        self.dict[lst[1]] = lst[2]
                    else:
                        raise CatalogFormatError()

    def load_file(self, file_name):
        try:
            file_object = open(file_name, mode='r', encoding='utf-8')
            try:
                self.load(file_object)
            except IOError:
                print('ERROR: Can not read reference catalogue file!')
                return
            except CatalogFormatError:
                print('ERROR: Reference catalogue is damaged!')
                return
            finally:
                file_object.close()
        except IOError:
            print('ERROR: Can not open reference catalogue file!')

# not correct for unicode file names
class TarFileWriter:  # OSError, IOError, tarfile.TarError
    def __init__(self, name, max_part_size, arch_type='tar'):
        self.TarName = name
        self.PartNumber = 0
        self.PartSize = 0
        self.PartFile = None
        self.Closed = True
        self.MaxPartSize = (max_part_size // tarfile.RECORDSIZE) * tarfile.RECORDSIZE
        self.Type = arch_type.lower()
        if arch_type == 'tar':
            self.Ext = STR_TAR_EXT
            self.Mode = 'w:'
        elif arch_type == 'gz':
            self.Ext = STR_GZ_EXT
            self.Mode = 'w:gz'
        elif arch_type == 'bz2':
            self.Ext = STR_BZ2_EXT
            self.Mode = 'w:bz2'
        else:
            raise IOError()

    def close(self):  # IOError
        if not self.Closed:
            self.PartFile.close()
            self.PartFile = None
            self.Closed = True

    def __new_part(self):  # IOError
        self.close()
        self.PartNumber += 1
        self.PartFile = tarfile.open(self.TarName + STR_POINT + str(self.PartNumber) + self.Ext, self.Mode)
        self.PartSize = 0
        self.Closed = False

    def add(self, file_path, tar_name):  # OSError, IOError, tarfile.TarError
        if self.Closed:
            self.__new_part()
        # prepare file object
        file_size = os.path.getsize(file_path)  # OSError
        file_tar_info = self.PartFile.gettarinfo(file_path)  # tarfile.TarError
        file_tar_info.name = tar_name

        with open(file_path, 'rb') as file_object:  # IOError
            # copy file to tar
            while (self.PartSize + file_size + 3*tarfile.BLOCKSIZE) > self.MaxPartSize:
                file_size_to_save = self.MaxPartSize - self.PartSize - 3*tarfile.BLOCKSIZE
                file_tar_info.size = file_size_to_save
                self.PartFile.addfile(file_tar_info, file_object)  # tarfile.TarError
                self.PartSize = self.PartSize + tarfile.BLOCKSIZE + file_size_to_save
                assert (self.PartSize + 2*tarfile.BLOCKSIZE) == self.MaxPartSize
                self.__new_part()
                file_size -= file_size_to_save

            file_tar_info.size = file_size
            self.PartFile.addfile(file_tar_info, file_object)  # tarfile.TarError
            # recalculate PartSize
            self.PartSize = self.PartSize + tarfile.BLOCKSIZE + (file_size // tarfile.BLOCKSIZE) * tarfile.BLOCKSIZE
            if (file_size % tarfile.BLOCKSIZE) > 0:
                self.PartSize += tarfile.BLOCKSIZE

        assert (self.PartSize + 2*tarfile.BLOCKSIZE) <= self.MaxPartSize

        if (self.PartSize + 3*tarfile.BLOCKSIZE) >= self.MaxPartSize:
            self.close()

# not correct for unicode file names
class TarFileReader:  # KeyError, IOError, tarfile.TarError
    def __init__(self, name):
        self.TarName = name
        self.PartNumber = 0
        self.PartFile = None
        self.Closed = True
        if os.path.isfile(name + '.1' + STR_TAR_EXT):
            self.Ext = STR_TAR_EXT
        elif os.path.isfile(name + '.1' + STR_GZ_EXT):
            self.Ext = STR_GZ_EXT
        elif os.path.isfile(name + '.1' + STR_BZ2_EXT):
            self.Ext = STR_BZ2_EXT
        else:
            raise IOError()

    def close(self):  # IOError
        if not self.Closed:
            self.PartFile.close()
            self.PartFile = None
            self.Closed = True

    def __next_part(self):  # IOError
        self.close()
        self.PartNumber += 1
        self.PartFile = tarfile.open(self.TarName + STR_POINT + str(self.PartNumber) + self.Ext)

    def extract(self, tar_name, file_path):  # KeyError, IOError, tarfile.TarError
        self.PartNumber = 0

        # ищем первый том в котором есть такой файл
        found = False
        no_file = False
        while not (found or no_file):
            try:
                self.__next_part()
                file_tar_info = self.PartFile.getmember(tar_name)
                found = True
            except IOError:
                no_file = True
            except KeyError:
                pass

        if found:
            with open(file_path, 'wb') as file_object:  # IOError
                while found:
                    # копируем в файл
                    tar_buffer = self.PartFile.extractfile(file_tar_info)  # tarfile.TarError
                    file_size = file_tar_info.size
                    while file_size > 0:
                        if file_size > tarfile.BLOCKSIZE:
                            file_size_to_save = tarfile.BLOCKSIZE
                        else:
                            file_size_to_save = file_size
                        file_object.write(tar_buffer.read(tarfile.BLOCKSIZE))  # IOError, tarfile.TarError
                        file_size = file_size - file_size_to_save
                    tar_buffer.close()  # tarfile.TarError
                    # проверяем в следующем томе
                    try:
                        self.__next_part()
                        file_tar_info = self.PartFile.getmember(tar_name)  # tarfile.TarError
                    except IOError:
                        found = False
                    except KeyError:
                        found = False
        else:
            raise KeyError()

def sh_create(sh_args):
    # check source
    if not os.path.isdir(sh_args.source):
        print('ERROR: Source not found!')
        return

    # check repository
    if not os.path.isdir(sh_args.repository):
        print('ERROR: Repository not found!')
        return

    # check if files with backup name exist
    if os.path.isfile(sh_args.repository + STR_SLASH + sh_args.name + STR_CAT_EXT):
        print('ERROR: Such archive already exists!')
        return

    # create empty reference and hash lists
    reference_list = FileList()
    hash_list = HashList()

    # load reference and hash lists
    if sh_args.reference is not None:
        # check if reference file exists
        ref_path = sh_args.repository + '/' + sh_args.reference + STR_CAT_EXT
        if not os.path.isfile(ref_path):
            print('ERROR: Reference not found!')
            return
        reference_list.load_file(ref_path)
        hash_list.load_file(ref_path)

    # create list of files/dirs in source destination
    source_list = FileList()
    source_list.read_dir_list(sh_args.source)

    # include / exclude files / dirs
    source_list.include_hierarchy(sh_args.include)
    source_list.exclude(sh_args.exclude)

    # compression
    compr = 'tar'
    if sh_args.compression is not None:
        compr = sh_args.compression

    # create TarFileWriter
    writer = TarFileWriter(sh_args.repository + STR_SLASH + sh_args.name, sh_args.size, compr)
    # check files and if new/changed add to archive
    c_all = 0
    c_new = 0
    size_all = 0
    size_new = 0
    key_list = list(source_list.dict)
    key_list.sort()
    for file_name in key_list:
        file_path = sh_args.source + file_name
        if not source_list.dict[file_name].isDir:
            ok = False
            while not ok:
                try:
                    # get date and size
                    source_list.dict[file_name].mtime = int(os.path.getmtime(file_path))
                    source_list.dict[file_name].size = os.path.getsize(file_path)
                    # check if such file is in reference
                    if (not sh_args.recalculate) and (file_name in reference_list.dict) and \
                            (not reference_list.dict[file_name].isDir) and \
                            (source_list.dict[file_name].mtime == reference_list.dict[file_name].mtime) and \
                            (source_list.dict[file_name].size == reference_list.dict[file_name].size):
                        source_list.dict[file_name].hash = reference_list.dict[file_name].hash
                    else:
                        # calculate hash
                        source_list.dict[file_name].hash = calc_hash(file_path)
                        # add file to archive
                        tar_name = hash_name(source_list.dict[file_name])
                        if tar_name not in hash_list.dict:
                            hash_list.dict[tar_name] = sh_args.name
                            writer.add(sh_args.source + file_name, tar_name)
                            c_new += 1
                            size_new = size_new + source_list.dict[file_name].size
                    size_all = size_all + source_list.dict[file_name].size
                    ok = True
                except (OSError, IOError) as e:
                    print('ERROR: Can not read: ' + e.filename)
                    if sh_args.ignore:
                        answer = 'i'
                    else:
                        answer = input('Abort (a) / Ignore (i) / Retry (other): ')
                    if answer == 'a':
                        writer.close()
                        return
                    elif answer == 'i':
                        del source_list.dict[file_name]
                        ok = True
                except tarfile.TarError:
                    print('ERROR: Can not write files to archive!')
                    answer = input('Abort (a) / Retry (other): ')
                    if answer == 'a':
                        writer.close()
                        return
            c_all += 1
        if not sh_args.quiet:
            sys.stdout.write("\rFiles (New/All): %s / %s, Size (New/All): %.02f Mb / %.02f Mb" % (
                             c_new, c_all, size_new/1024.0/1024.0, size_all/1024.0/1024.0))
            sys.stdout.flush()

    # close TarFileWriter
    writer.close()

    if not sh_args.quiet:
        sys.stdout.write(STR_EOL)
        sys.stdout.flush()

    # save catalogue
    try:
        file_object = open(sh_args.repository + STR_SLASH + sh_args.name + STR_CAT_EXT,
                           mode='w', encoding='utf-8')
        try:
            source_list.save(file_object)
            hash_list.save(file_object)
        except IOError:
            print('ERROR: Can not create catalogue file!')
            return
        finally:
            file_object.close()
    except IOError:
        print('ERROR: Can not create catalogue file!')

def sh_find(sh_args):
    # check repository
    if not os.path.isdir(sh_args.repository):
        print('ERROR: Repository not found!\n')
        return

    # get file list
    cat_list = os.listdir(sh_args.repository)
    cat_list.sort()
    key_list = list(cat_list)
    for key in key_list:
        if not fnmatch.fnmatch(key, sh_args.name + STR_CAT_EXT):
            del cat_list[cat_list.index(key)]

    # check if something found
    if len(cat_list) == 0:
        print('ERROR: No catalogue found!\n')
        return

    # looking for patterns in all catalogues
    for cat in cat_list:
        # loading catalogue
        file_list = FileList()
        file_list.load_file(sh_args.repository + STR_SLASH + cat)

        # include / exclude files / dirs
        file_list.include(sh_args.include)
        file_list.exclude(sh_args.exclude)

        # looking for matching files and dirs
        key_list = list(file_list.dict.keys())
        key_list.sort()
        for key in key_list:
            print(cat + ': ' + key)

def sh_restore(sh_args):
    # check repository
    if not os.path.isdir(sh_args.repository):
        print('ERROR: Repository not found!\n')
        return

    # check existence of catalogue file
    if not os.path.isfile(sh_args.repository + STR_SLASH + sh_args.name + STR_CAT_EXT):
        print('ERROR: Catalogue not found!\n')
        return

    # check destination existence
    if not os.path.isdir(sh_args.destination):
        print('ERROR: Destination not found!\n')
        return

    # read FileList and HashList from catalogue
    source_list = FileList()
    hash_list = HashList()
    source_list.load_file(sh_args.repository + STR_SLASH + sh_args.name + STR_CAT_EXT)
    hash_list.load_file(sh_args.repository + STR_SLASH + sh_args.name + STR_CAT_EXT)

    # include / exclude files / dirs
    source_list.fix_hierarchy()
    source_list.include_hierarchy(sh_args.include)
    source_list.exclude(sh_args.exclude)

    # create not existing dirs and extract new or changed files
    c_all = 0
    c_new = 0
    size_all = 0
    size_new = 0
    key_list = list(source_list.dict)
    key_list.sort()
    for file_name in key_list:
        file_path = sh_args.destination + file_name
        # make directory
        if source_list.dict[file_name].isDir:
            file_dir = file_path
        else:
            (file_dir, stub) = os.path.split(file_path)
        ok = False
        while not ok:
            try:
                if os.path.isfile(file_dir):
                    os.remove(file_dir)
                if not os.path.isdir(file_dir):
                    os.makedirs(file_dir)
                ok = True
            except OSError as e:
                print('ERROR: Can not create directory: ' + e.filename)
                if sh_args.ignore:
                    answer = 'i'
                else:
                    answer = input('Abort (a) / Ignore (i) / Retry (other): ')
                if answer == 'a':
                    return
                elif answer == 'i':
                    ok = True
        # restore file
        if not source_list.dict[file_name].isDir:
            hash_key = hash_name(source_list.dict[file_name])
            backup_file = hash_list.dict[hash_key]
            ok = False
            while not ok:
                try:
                    # check if such file exists
                    reader = TarFileReader(sh_args.repository + STR_SLASH + backup_file)
                    if os.path.isfile(file_path) and \
                            (source_list.dict[file_name].mtime == int(os.path.getmtime(file_path))) and \
                            (source_list.dict[file_name].size == os.path.getsize(file_path)) and \
                            (source_list.dict[file_name].hash == calc_hash(file_path)):
                        pass
                    else:
                        if os.path.isdir(file_path):
                            shutil.rmtree(file_path)
                        reader.extract(hash_key, file_path)
                        c_new += 1
                        size_new = size_new + source_list.dict[file_name].size
                    ok = True
                except (OSError, IOError) as e:
                    print('ERROR: Can not restore file: ' + e.filename)
                    if sh_args.ignore:
                        answer = 'i'
                    else:
                        answer = input('Abort (a) / Ignore (i) / Retry (other): ')
                    if answer == 'a':
                        return
                    elif answer == 'i':
                        ok = True
                finally:
                    reader.close()
            c_all += 1
            size_all = size_all + source_list.dict[file_name].size
        # set time
        ok = False
        while not ok:
            try:
                os.utime(file_path, (source_list.dict[file_name].mtime,
                         source_list.dict[file_name].mtime))
                ok = True
            except OSError as e:
                print('ERROR: Can not update time for: ' + e.filename)
                if sh_args.ignore:
                    answer = 'i'
                else:
                    answer = input('Abort (a) / Ignore (i) / Retry (other): ')
                if answer == 'a':
                    return
                elif answer == 'i':
                    ok = True
        sys.stdout.write("\rFiles (New/All): %s / %s, Size (New/All): %.02f Mb / %.02f Mb" % (
                         c_new, c_all, size_new/1024.0/1024.0, size_all/1024.0/1024.0))
        sys.stdout.flush()

    sys.stdout.write(STR_EOL)
    sys.stdout.flush()

    # get FileList for destination
    if sh_args.delete:
        destination_list = FileList()
        destination_list.read_dir_list(sh_args.destination)
        # remove old files
        key_list = list(destination_list.dict.keys())
        key_list.sort()
        for file_name in key_list:
            file_path = sh_args.destination + file_name
            if (not destination_list.dict[file_name].isDir) and \
                    (file_name not in source_list.dict):
                ok = False
                while not ok:
                    try:
                        os.remove(file_path)
                        ok = True
                    except OSError as e:
                        print('ERROR: Can not delete file: ' + e.filename)
                        if sh_args.ignore:
                            answer = 'i'
                        else:
                            answer = input('Abort (a) / Ignore (i) / Retry (other): ')
                        if answer == 'a':
                            return
                        elif answer == 'i':
                            ok = True
        # remove old dirs
        key_list = list(destination_list.dict.keys())
        key_list.sort()
        for file_name in key_list:
            file_path = sh_args.destination + file_name
            if destination_list.dict[file_name].isDir and \
                    (file_name not in source_list.dict):
                ok = False
                while not ok:
                    try:
                        if os.path.isdir(file_path):
                            shutil.rmtree(file_path)
                        ok = True
                    except OSError as e:
                        print('ERROR: Can not delete directory: ' + e.filename)
                        if sh_args.ignore:
                            answer = 'i'
                        else:
                            answer = input('Abort (a) / Ignore (i) / Retry (other): ')
                        if answer == 'a':
                            return
                        elif answer == 'i':
                            ok = True

# source - папка, которая архивируется
# destination - папка, в которую извлекается
# repository - папка в которой хранится архив
# name - имя архива (без расширения и без пути, архив должен быть в [repository])
# reference - путь/имя каталога эталона с расширением
# slice - том архива, перед расширением номер [backup].3.tar
# catalogue - каталог архива - [backup].cat
parser = argparse.ArgumentParser(description='version 0.6.2')
subparsers = parser.add_subparsers()

parser_create = subparsers.add_parser('create')  #
parser_create.add_argument('source', help='Directory tree that will be backed up.')  # dir
parser_create.add_argument('repository', help='Directory in which backup will be stored.')  # dir
parser_create.add_argument('name', help='Basename for backup.')  # name
parser_create.add_argument('-r', '--reference',
                           help='Reference basename for differential backup. '
                                'Reference catalog should be stored in the same repository.')  # path
parser_create.add_argument('-s', '--size', type=int, default=1024*1024*1020, help='Size of one slice.')
parser_create.add_argument('-i', '--include', nargs='*',
                           help='Mask list. Files/Dirs matching at least one mask will be included in backup. '
                                'If no mask specified all Files/Dirs will be included.')
parser_create.add_argument('-e', '--exclude', nargs='*',
                           help='Mask list. Files/Dirs matching at least one mask will be excluded from backup.')
parser_create.add_argument('-q', '--quiet', action='store_true',
                           help='Nothing is displayed if operation succeeds.')  # !!!
parser_create.add_argument('-g', '--ignore', action='store_true', help='Ignore all errors.')
parser_create.add_argument('-c', '--compression', help="'tar'-default, 'gz' or 'bz2'")
parser_create.add_argument('-a', '--recalculate', action='store_true',
                           help="Recalculate all hashes again. Don't use hashes from reference.")
parser_create.set_defaults(func=sh_create)

parser_find = subparsers.add_parser('find')  # simple regular expressions
parser_find.add_argument('repository', help='Directory in which backup is stored.')  # dir
parser_find.add_argument('name', help='Mask for backup basename. '
                                      'Several backups can be looked thorough.')  # name pattern (without ext)
parser_find.add_argument('-i', '--include', nargs='*',
                         help='Mask list. Files/Dirs matching at least one mask will be shown. '
                              'If no mask specified all Files/Dirs will be shown.')
parser_find.add_argument('-e', '--exclude', nargs='*',
                         help='Mask list. Files/Dirs matching at least one mask will not be shown.')
parser_find.set_defaults(func=sh_find)

parser_restore = subparsers.add_parser('restore')  # restore backup
parser_restore.add_argument('repository', help='Directory in which backup is stored.')  # dir
parser_restore.add_argument('name', help='Basename for backup to be restored.')  # name
parser_restore.add_argument('destination', help='Directory which will be restored.')  # dir
parser_restore.add_argument('-i', '--include', nargs='*',
                            help='Mask list. Files/Dirs matching at least one mask will be restored. '
                                 'If no mask specified all Files/Dirs will be restored.')
parser_restore.add_argument('-e', '--exclude', nargs='*',
                            help='Mask list. Files/Dirs matching at least one mask will not be restored.')
parser_restore.add_argument('-d', '--delete', action='store_true',
                            help='Delete Files/Dirs not existing in backup.')
parser_restore.add_argument('-g', '--ignore', action='store_true', help='Ignore all errors.')
parser_restore.set_defaults(func=sh_restore)

args = parser.parse_args()
args.func(args)

# // целочисленное деление, результат – целое число (дробная часть отбрасывается)
# % деление по модулю

# tar file format
# 1 file info - BLOCKSIZE (512)
# 1 file data - filled by zeros to BLOCKSIZE (512)
# 2 file info - BLOCKSIZE (512)
# 2 file data - filled by zeros to BLOCKSIZE (512)
# N file info - BLOCKSIZE (512)
# N file data - filled by zeros to BLOCKSIZE (512)
# two finishing zero blocks - BLOCKSIZE * 2 (512 * 2)
# filled by zeros to RECORDSIZE (BLOCKSIZE * 20) (512 * 20)
# tarfile.BLOCKSIZE = 512
# tarfile.RECORDSIZE = BLOCKSIZE * 20


#!/usr/bin/env python3
"""This modual holds the class created for task 3"""
import numpy as np
import matplotlib.pyplot as plt

class Neuron:
    """
        Neuron - class for a neuron
        nx = the number of input freatures to the neuron
    """
    def __init__(self, nx):
        if not isinstance(nx, int):
            raise TypeError("nx must be an integer")
        if nx < 1:
            raise ValueError("nx must be a positive integer")
        self.__W = np.random.randn(1, nx)
        self.__b = 0
        self.__A = 0

    @property
    def W(self):
        return self.__W

    @property
    def b(self):
        return self.__b

    @property
    def A(self):
        return self.__A

    def forward_prop(self, X):
        """
        This function calculates the forward propogation
        and updates the private attibute A
        """
        # X numpy array size = (nx - , m - )
        Z = np.dot(self.W, X) + self.b
        # Z = the (weight*activation)+bias for all data in the set

        A = 1/(1 + np.exp(-1 * Z))
        # applying the sigmoid function to Z (3brown1blue need to rewatch)

        self.__A = A
        return self.__A

    def cost(self, Y, A):
        """Calculates the cost of the model using logistic regression"""
        m = Y.shape[1]
        loss = -1 * (Y * np.log(A) + (1 - Y) * np.log(1.0000001 - A))
        cost = (1/m) * np.sum(loss)
        return cost

    def evaluate(self, X, Y):
        """Evalueates neurons predictions"""
        predict = self.forward_prop(X)
        predict = np.where(predict < 0.5, 0, 1)
        return predict, self.cost(Y, self.__A)

    def gradient_descent(self, X, Y, A, alpha=0.05):
        """
            - Calculates one pass of gradient descent on the neuron
            - Gradient decent updates the weights and biases
        """
        m = Y.shape[1]
        W = self.__W
        b = self.__b

        Dz = A - Y
        Dw = (1/m) * (Dz @ X.T)
        Db = (1/m) * np.sum(Dz)

        self.__W = W - alpha * Dw
        self.__b = b - alpha * Db

    def train(
            self, X, Y,
            iterations=5000, alpha=0.05, verbose=True,
            graph=True, step=100):
        """Trains a neuron"""
        c = self.cost
        if not isinstance(iterations, int):
            raise TypeError("iterations must be an integer")
        if iterations < 0:
            raise ValueError("iterations must be a positive integer")
        if not isinstance(alpha, float):
            raise TypeError("alpha must be a float")
        if alpha < 0:
            raise ValueError("alpha must be positive")
        if verbose or graph:
            if not isinstance(step, int):
                raise TypeError("step must be an integer")
            if step <= 0 or step > iterations:
                raise ValueError("step must be positive and <= iterations")
        it = []
        cost = []
        for i in range(iterations + 1):
            A = self.forward_prop(X)
            if step == 0 or i % step == 0 or i == iterations:
                if verbose:
                    print("Cost after {} iterations: {}".format(i, c(Y, A)))
                if graph:
                    it.append(i)
                    cost.append(self.cost(Y, A))
            self.gradient_descent(X, Y, A, alpha)

        it = np.array(it)
        cost = np.array(cost)
        if graph:
            plt.plot(it, cost)
            plt.xlabel("iteration")
            plt.ylabel("cost")
            plt.title("Training Cost")
            plt.show()

        return self.evaluate(X, Y)