import requests
from typing import Any, Text, Dict, List
from rasa_sdk import Action, Tracker
from rasa_sdk.executor import CollectingDispatcher

class ActionAskCovidRegion(Action):

    def name(self) -> Text:
        return "action_info_covid_regioni"

    def run(self, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        response = requests.get(
            "https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-json/dpc-covid19-ita-regioni-latest.json").json()
        entity_par=tracker.latest_message["entities"] #recupero delle entities contenute nell'ultimo messaggio
        check_slot = tracker.get_slot("regione") #recupero dello slot
        regione = ""
        parameter = ""
        print("last message x par:", entity_par, "\n")
        print(check_slot)

        if check_slot is not None:
            regione=check_slot
        message = "please enter a correct italian region name"

        for e in entity_par:
            if e['entity'] == "parameter":
                parameter = e['value']
                parameter= parameter.strip().title() #recupero parametro

        for data in response: #costruzione del messaggio output
            if data["denominazione_regione"] == regione.title():
                print(data)
                if parameter == "Actual Positive Cases":
                    message="Actual positive cases: " + str(data['nuovi_positivi'])
                elif parameter == "Hospitalized With Symptoms":
                    message = "Hospitalized with symptoms: " + str(data['ricoverati_con_sintomi'])
                elif parameter == "Intensive Care Unit":
                    message = "Intensive care unit: " + str(data['terapia_intensiva'])
                elif parameter == "Total Deceased":
                    message = "Total deceased: " + str(data['deceduti'])
                elif parameter == "General Situation":
                    message = "Actual positive cases: " + str(data['nuovi_positivi']) + " Hospitalized with symptoms: " + str(
                    data['ricoverati_con_sintomi']) + " Intensive care unit:" + str(
                    data['terapia_intensiva']) + " Total deceased:" + str(data['deceduti'])
                else: print("it wasnt possible to provide datas, sorry")

        print(message)
        dispatcher.utter_message( regione + ": \n" + message) #presentazione del messaggio

        return []

class ActionAskCovidItaly(Action):

    def name(self) -> Text:
        return "action_info_covid_ita"

    def run(self, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        global parameter
        response = requests.get(
            "https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-json/dpc-covid19-ita-andamento-nazionale-latest.json").json()
        intent = tracker.latest_message["intent"]
        entity_par = tracker.latest_message["entities"]
        message1=""
        print("last message", intent)
        print("last message x par:", entity_par, "\n")

        for e in entity_par:
            if e['entity'] == "parameter":
                parameter = e['value']
                parameter= parameter.strip().title()

        for data in response:
            if data["stato"] == "ITA":
                print(data)
                if parameter == "Actual Positive Cases":
                    message1 = "Actual positive cases: " + str(data['nuovi_positivi'])
                elif parameter == "Hospitalized With Symptoms":
                    message1 = "Hospitalized with symptoms: " + str(data['ricoverati_con_sintomi'])
                elif parameter == "Intensive Care Unit":
                    message1 = "Intensive care unit: " + str(data['terapia_intensiva'])
                elif parameter == "Total Deceased":
                    message1 = "Total deceased: " + str(data['deceduti'])
                elif parameter == "General Situation":
                    message1 = "Actual positive cases: " + str(
                        data['nuovi_positivi']) + " Hospitalized with symptoms: " + str(
                        data['ricoverati_con_sintomi']) + " Intensive care unit:" + str(
                        data['terapia_intensiva']) + " Total deceased:" + str(data['deceduti'])
                else:
                    print("it wasnt possible to provide datas, sorry")

        print(message1)
        dispatcher.utter_message("Covid situation in Italy: "  + "\n" + message1)

        return []


import argparse
import os.path
import pandas
import plotly.express as px
from tkinter import *
from PIL import ImageTk,Image

VERSION = "0.1"

# Start
# Main Function
def gui():
    main_window = Tk()
    # size of the window
    main_window.geometry('670x480')
    main_window.title("Threat Calculator")
    # Menu Declaration
    menu = Menu(main_window)
    main_window.config(menu=menu)
    # Restart Program Function
    def restart_program():
        python = sys.executable
        os.execl(python, python, *sys.argv)
    menu.add_command(label="Main", command=restart_program)

    # Create Tkinter variables
    oppVar = StringVar(main_window)
    intVar = StringVar(main_window)
    capVar = StringVar(main_window)

    # Define Frames
    contentframe = LabelFrame(main_window, padx=5, pady=5)
    contentframe.grid(column=3, row=0, columnspan=5, rowspan=8, pady=15)
    resultframe = LabelFrame(main_window, padx=5, pady=5)
    resultframe.grid(row=7, column=0, columnspan=2)

    # Calc Function
    def calc():
        # Evaluate Intent
        intresult = 0
        if intVar.get() == "Low":
            intresult = 0
        elif intVar.get() == "Moderate":
            intresult = 1
        elif intVar.get() == "High":
            intresult = 2
        else:
            intresult = 3

        # Evaluate Opportunity
        oppresult = 0
        if oppVar.get() == "Low":
            oppresult = 0
        elif oppVar.get() == "Moderate":
            oppresult = 1
        elif oppVar.get() == "High":
            oppresult = 2
        else:
            oppresult = 3

        # Evaluate Capability
        capresult = 0
        if capVar.get() == "Low":
            capresult = 0
        elif capVar.get() == "Moderate":
            capresult = 1
        elif capVar.get() == "High":
            capresult = 2
        else:
            capresult = 3

        # Evaluate Severity based on criteria
        result = capresult + intresult + oppresult
        if result <= 2:
            severity = "Low"
        elif 2 < result <= 4:
            severity = "Moderate"
        elif 4 < result <= 6:
            severity = "High"
        else:
            severity = "Critical"

        # Evaluate Threat Nature
        if intresult > 1 and oppresult > 1 and capresult > 1:
            theat_type = "Imminent"
        elif intresult > 0 and oppresult > 0 and capresult <= 1:
            theat_type = "Insubstantial"
        elif intresult > 0 and capresult > 1:
            theat_type = "Impending"
        elif intresult == 0 and oppresult > 1 and capresult > 1:
            theat_type = "Potential"
        else:
            theat_type = "Nonexistent"
        final_result = "There is a " + severity + " " + theat_type + " Threat"

        #Output result to frame
        resultframe.config(text=final_result, font=("Helvetica", 9, "bold"))

    # Add image
    img = ImageTk.PhotoImage(Image.open("model.png"))
    Label(contentframe, image=img).grid()

    #row 1
    oppLabel = Label(main_window, text="Opportunity", font=("Helvetica", 12, "bold"), anchor=W, justify=LEFT)
    oppLabel.grid(row=0, column=0)

    # Dictionary with options
    oppchoices = ['Low', 'Moderate', 'High', 'Critical']
    oppVar.set('Low')  # set the default option
    popupMenu = OptionMenu(main_window, oppVar, *oppchoices)
    popupMenu.config(width=20)
    Label(main_window, text="Choose a rating").grid(row=0, column=1, pady=10)
    popupMenu.grid(row=1, column=0, columnspan=2)

    # row 2
    intLabel = Label(main_window, text="Intent", font=("Helvetica", 12, "bold"), anchor=W, justify=LEFT)
    intLabel.grid(row=2, column=0)

    # Dictionary with options
    intchoices = ['Low', 'Moderate', 'High', 'Critical']
    intVar.set('Low')  # set the default option
    popupMenu = OptionMenu(main_window, intVar, *intchoices)
    popupMenu.config(width=20)
    Label(main_window, text="Choose a rating").grid(row=2, column=1, pady=10)
    popupMenu.grid(row=3, column=0, columnspan=2)

    # row 3
    capLabel = Label(main_window, text="Capability", font=("Helvetica", 12, "bold"), anchor=W, justify=LEFT)
    capLabel.grid(row=4, column=0)

    # Dictionary with options
    capchoices = ['Low', 'Moderate', 'High', 'Critical']
    capVar.set('Low')  # set the default option
    popupMenu = OptionMenu(main_window, capVar, *capchoices)
    popupMenu.config(width=20)
    Label(main_window, text="Choose a rating").grid(row=4, column=1,  pady=10)
    popupMenu.grid(row=5, column=0, columnspan=2)

    #submit button
    submitButton = Button(main_window, text="Submit", font=("bold"), width=20, command=calc)
    submitButton.grid(row=6, column=0, columnspan=2, padx=10, pady=20)

    #Display Results
    Label(resultframe).grid(row=7, column=0, columnspan=2)

    main_window.mainloop()

if __name__ == '__main__':
    gui()


from typing import Optional

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    def swapHelper(self, head: Optional[ListNode],root: Optional[ListNode], prev:Optional[ListNode]):
        if head == None:
            return root
        elif head.next == None:
            return root
        else:
            third = head.next.next
            temp = head
            head = head.next
            head.next = temp
            temp.next = third
            prev.next = head
            return self.swapHelper(third,root,head.next)

    def swapPairs(self, head: Optional[ListNode]) -> Optional[ListNode]:
        if head == None:
            return None
        elif head.next == None:
            return head
        else:
            third = head.next.next
            temp = head
            head = head.next
            head.next = temp
            temp.next = third
            return self.swapHelper(third,head,head.next)

#Exercise: https://leetcode.com/problems/swap-nodes-in-pairs/
#Given a linked list, swap every two adjacent nodes and return
# its head. You must solve the problem without modifying the
# values in the list's nodes (i.e., only nodes themselves may be changed.)


import bluetooth
import struct

def connect_bluetooth(mac: str):
	try:
		print(f"Connecting to MAC address {mac}")
		sock = bluetooth.BluetoothSocket(bluetooth.RFCOMM)
		sock.connect((mac, 1))
		print(f"Connected to {mac}")

		return sock

	except bluetooth.btcommon.BluetoothError as e:
		print(e)
		return None

class BluetoothWriter:
	def __init__(self, sock):
		self.__socket = sock

	# opcode and params must be 0-127
	def write(self, opcode: int, *params: int):
		integers = [opcode, *params]
		print(integers)
		data = "".join(chr(c) for c in integers)
		self.__socket.send(data)


import pygame
import math
import sys

class GUI:
    def __init__(self, service, multiplayer):
        self._service = service
        self.__multiplayer = multiplayer
        self.board = self._service.repo.table
        self.__columns = self._service.repo.number_columns
        self.__rows = self._service.repo.number_rows

        self.blue = (0, 0, 255)
        self.black = (0, 0, 0)
        self.red = (255, 0, 0)
        self.yellow = (255, 255, 0)
        self.square_size = 100
        self.radius = int(self.square_size / 2 - 5)
        self.width = self.__rows * self.square_size
        self.height = (self.__columns + 1) * self.square_size
        self.size = (self.width, self.height)
        self.screen = pygame.display.set_mode(self.size)

        self.game_over = False

    def print_table(self):
        print(self.board)

    def outcome_label(self, my_font, n):
        if self._service.check_win() is True:
            label = my_font.render(n + " wins!!", True, self.yellow)
            self.screen.blit(label, (40, 10))
            self.game_over = True
        if self._service.check_draw() is True:
            label = my_font.render("Draw!!", True, self.red)
            self.screen.blit(label, (40, 10))
            self.game_over = True

    def players(self, event, moves, my_font):
        posix = event.pos[0]
        col = int(math.floor(posix / self.square_size))
        m, n = 0, 0
        for move in moves:
            if move[1] == col:
                m = move[0]
                n = move[1]

        if self._service.validate_move(m, n):
            self._service.move_player(m, n, 2)

            if self._service.check_win() is True:
                label = my_font.render("Player 2 wins!!", True, self.yellow)
                self.screen.blit(label, (40, 10))
                self.game_over = True
            if self._service.check_draw() is True:
                label = my_font.render("Draw!!", True, self.red)
                self.screen.blit(label, (40, 10))
                self.game_over = True

    def draw_board(self):
        for c in range(self.__rows):
            for r in range(self.__columns):
                pygame.draw.rect(self.screen, self.blue, (
                    c * self.square_size, r * self.square_size + self.square_size, self.square_size, self.square_size))
                pygame.draw.circle(self.screen, self.black, (
                    int(c * self.square_size + self.square_size / 2),
                    int(r * self.square_size + self.square_size + self.square_size / 2)), self.radius)

        for c in range(self.__rows):
            for r in range(self.__columns):
                if self.board.value(r, c) == 1:
                    pygame.draw.circle(self.screen, self.red, (
                        int(c * self.square_size + self.square_size / 2),
                        self.height - int((self.__columns - r - 1) * self.square_size + self.square_size / 2)),
                                       self.radius)
                elif self.board.value(r, c) == 2:
                    pygame.draw.circle(self.screen, self.yellow, (
                        int(c * self.square_size + self.square_size / 2),
                        self.height - int((self.__columns - r - 1) * self.square_size + self.square_size / 2)),
                                       self.radius)
        pygame.display.update()

    def start(self):
        pygame.init()

        my_font = pygame.font.SysFont("monospace", 75)
        turn = 0
        self.draw_board()
        pygame.display.update()
        while not self.game_over:

            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    sys.exit()

                if event.type == pygame.MOUSEMOTION:
                    pygame.draw.rect(self.screen, self.black, (0, 0, self.width, self.square_size))
                    posix = event.pos[0]
                    if turn == 0:
                        pygame.draw.circle(self.screen, self.red, (posix, int(self.square_size / 2)), self.radius)
                    else:
                        pygame.draw.circle(self.screen, self.yellow, (posix, int(self.square_size / 2)), self.radius)
                pygame.display.update()

                if event.type == pygame.MOUSEBUTTONDOWN:
                    pygame.draw.rect(self.screen, self.black, (0, 0, self.width, self.square_size))
                    moves = self._service.repo.possible_moves()

                    # Ask for Player 1 Input
                    if turn == 0:
                        posix = event.pos[0]
                        col = int(math.floor(posix / self.square_size))
                        m, n = 0, 0
                        for move in moves:
                            if move[1] == col:
                                m = move[0]
                                n = move[1]
                        if self._service.validate_move(m, n):
                            self._service.move_player(m, n, 1)

                            if self._service.check_win() is True:
                                label = my_font.render("Player 1 wins!!", True, self.red)
                                self.screen.blit(label, (40, 10))
                                self.game_over = True
                                self.__multiplayer = True
                                turn += 1

                            if self._service.check_draw() is True:
                                label = my_font.render("Draw!!", True, self.red)
                                self.screen.blit(label, (40, 10))
                                self.game_over = True
                            if self.__multiplayer is False:
                                self._service.move_brain()
                                self.outcome_label(my_font, "Computer")
                                turn += 1

                    # Ask for Player 2 Input
                    else:
                        self.players(event, moves, my_font)

                    # self.print_table()
                    self.draw_board()

                    turn += 1
                    turn = turn % 2

                    if self.game_over:
                        pygame.time.wait(3000)


from django import forms  #import modułu wbudowanych formularzy

from .models import Topic, Entry #import modelu Topic

class TopicForm(forms.ModelForm): #Utworzenie klasy TopicForm z przekazanym modelem formularza
    class Meta:
        model = Topic #Budowanie formularza o model Topic
        fields = ['text'] #Możliwość wprowadzania tylko treść do pola text
        labels = {'text': ''} #informacja, że kolumna text nie ma etykiety

class EntryForm(forms.ModelForm):
    class Meta:
        model = Entry
        fields = ['text']
        labels = {'text': ''}
        widgets = {'text': forms.Textarea(attrs={'cols': 80})} #Nadpisanie ustawień aby dla kolumny 'text' wyświetlic textarea o szerokości 80


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# Train the model
# After training we will have a dataset
# dataset is sent to CNN classifier we have used (VGG16) here.
# Examples of other CNN classifier: Resnet, InceptionNet, MObile Net V2
# Mobile Net V2 is very light, It uses Rpi, 25 Mb, accuray is 70% wrt 72% of VGG16 whih uses 500 Mb

# In[2]:

# Mask problem is easy as compared to emotion problem.
# Here, Model is exported.
# We are using transfer learning, where last layer is removed and our layer is added.
# Input is taken using webcam that is applied to the model.
# Screen detection is done first.
# We are cropping the face only with the rectangle for improved accuracy.
# Haar cascade is used to get frontal face detection.

# In[3]:

import os
# to get all files

# In[4]:

from keras.preprocessing import image
import cv2

# In[5]:

categories = ['without_mask','with_mask']

# In[6]:

data = []
for category in categories:
    path = os.path.join('train',category)

    label = categories.index(category)
    # without_mask index is 0
    # with_mask index is 1

    for file in os.listdir(path):
        # Now we will read image using opencv

        img_path = os.path.join(path,file)
        img = cv2.imread(img_path)
        img = cv2.resize(img,(224,224))

        data.append([img,label]) # for adding both the with and without mask images
        # (image,0) for image in without_mask folder
                                 #(image,1) for image in with_mask folder

# we will create a dataset 2 columns, 1st one is image and 2d is the label

# In[7]:

len(data)

# In[8]:

import random

# In[9]:

random.shuffle(data) # for mixing the with and without mask images

# In[10]:

# to separate X and y and then convert it into numpy array
X = []
y = []

for features,label in data:
    X.append(features)
    y.append(label)

# In[11]:

len(X)

# In[12]:

len(y)

# In[13]:

import numpy as np

# In[14]:

X = np.array(X)
# X should be 2d array
y = np.array(y)
# y should be 1d array since it is just a label

# In[15]:

X.shape
# 1508 is no of images with its shape (224,224,3) 224x224 and 3 is no of channels

# In[16]:

y.shape
# y are the labels

# In[17]:

y

# In[18]:

X

# In[19]:

X = X/255# in order to standize the X

# In[20]:

from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2)

# In[21]:

X_test.shape

# In[22]:

# 302 is no of images in test

# In[23]:

# we will create CNN architecture and do the transfer learning

# In[24]:

from keras.applications.vgg16 import VGG16

# In[25]:

vgg = VGG16()

# In[26]:

vgg.summary()
# This is our vgg model

# In[27]:

# predictions (Dense)         (None, 1000)              4097000
#This last layer will be removed and we will add our layer with 1 node

# In[28]:

from keras import Sequential

# In[29]:

model = Sequential()

# In[30]:

for layer in vgg.layers[:-1]:  # to exclude last layer
    model.add(layer)

# In[31]:

model.summary()
# now last layer is removed and new model is created
# vgg is a functional model, but we needed sequential thats why we used sequential workflow

# In[32]:

#we will freeze all the layers so that their weights wont be updated during training

# In[33]:

for layer in model.layers:
    layer.trainable = False

# In[34]:

model.summary()

# In[35]:

# Now, all trainable parameters are set to 0 i.e. parameters are freezed

# In[36]:

# we are adding our last layer
from keras.layers import Dense

# In[37]:

model.add(Dense(1,activation='sigmoid'))

# In[38]:

model.summary()

# In[39]:

# this last layer has 4097 parameters, its 1 and 4096 coming from previous layer

# In[40]:

# 4097 are trainable parameters

# In[41]:

model.compile(optimizer='Adam',loss='binary_crossentropy',metrics=['accuracy'])
# to increase the accuracy, we can use learning rate of 0.0001

# In[ ]:

model.fit(X_train,y_train,epochs=5,validation_data=(X_test,y_test))

# In[ ]:

cap = cv2.VideoCapture(0) # to detect the face, 0 for webcam

# In[ ]:

while True:

    ret,frame = cap.read()
    #reading the image
    #ret gives us true or flase value
    #frame gives us the image

    # call for detection method
    img = cv2.resize(frame,(224,224))

    y_pred = detect_face_mask(img) # image is sent to the function, the ouput stored in y_pred
    # frame to get a smaller video, img for the normal window size

    coods = detect_face(cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY))

    for x,y,w,h in coods:
        cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2) # 1is thickness

    if y_pred == 0:
        draw_label(frame,"Mask",(30,30),(0,255,0))
    else:
        draw_label(frame,"No Mask",(30,30),(0,0,255))

    draw_label(frame,"Face Mask Detection", (10,10),(255,0,0))
    #(255,0,0) is blue color

    print(y_pred)

    cv2.imshow('windows', frame) # we are displaying the frame

    if cv2.waitKey(1) & 0xFF == ord('x'):
        break

cv2.destroyAllWindows()

# In[ ]:

def detect_face_mask(img):

    #y_pred = model.predict_classes(img.reshape(1,224,224,3))
    y_pred = model.predict(img.reshape(1,224,224,3))
    y_pred = np.round(y_pred).astype(int)
    # our trained model will predict whether it is with or without mask as 1 or 0 as output

    return y_pred

# In[ ]:

sample1 = cv2.imread('sample1.jpg')
sample1 = cv2.resize(sample1,(224,224))

# In[ ]:

detect_face_mask(sample1)

# In[ ]:

# 1 index means with mask, which is right.

# In[ ]:

# now implementing a text in the video telling us whether mask is on or not

def draw_label(img,text,pos,bg_color): #pos is position

    text_size = cv2.getTextSize(text,cv2.FONT_HERSHEY_SIMPLEX,1,cv2.FILLED) # 1 is font scale
    # cv2.filled is thickness

    end_x = pos[0]+text_size[0][0] + 2 # 2 is the margin
    end_y = pos[1]+text_size[0][1] - 2

    cv2.rectangle(img,pos,(end_x,end_y),bg_color,cv2.FILLED)
    cv2.putText(img,text,pos,cv2.FONT_HERSHEY_SIMPLEX,1,(0,0,0),1,cv2.LINE_AA)
    # (0,0,0) is the color, LINE_AA is type of line used

# In[ ]:

#face detection square

#import haar cascading file

haar = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')

def detect_face(img):

    coods = haar.detectMultiScale(img)
    return coods


import boto3
import time
import os
import csv
from datetime import datetime
import json

def main():
    event_bridge = boto3.client("events")

    data_s3_bucket_name = os.environ.get["S3_BUCKET_NAME"]
    data_s3_object_key = os.environ.get["S3_OBJECT_KEY"]

    if None == data_s3_bucket_name or None == data_s3_object_key:
        raise Exception(
            "S3_BUCKET_NAME and S3_OBJECT_KEY must be set in environment variables"
        )

    print("S3_BUCKET_NAME: " + data_s3_bucket_name)
    print("S3_OBJECT_KEY: " + data_s3_object_key)

    local_file = "/tmp/data.tsv"

    s3_resource = boto3.resource("s3")
    s3_resource.Object(data_s3_bucket_name, data_s3_object_key).download_file(
        local_file
    )

    print("Successfully downloaded file from S3" + local_file)

    with open(local_file) as csvfile:
        reader = csv.reader(csvfile, delimiter=",")
        headers = next(reader)

        for row in reader:
            print(",".join(row))
            event = {
                "status": "extracted",
                "headers": ",".join(headers),
                "data": ",".join(row),
            }
            response = event_bridge.put_events(
                Entries=[
                    {
                        "DetailType": "s3RecordExtraction",
                        "EventBusName": "default",
                        "Source": "eventbridge-s3-extraction-task",
                        "Time": datetime.now(),
                        "Detail": json.dumps(event),
                    }
                ]
            )
        exit(0)

if __name__ == "__main__":
    main()


import numpy as np
import csv
import keras
import tensorflow as tf

from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import Adadelta, SGD
from keras.callbacks import TensorBoard

config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
keras.backend.set_session(sess)

def loadData(dataPath):
	X = []
	Y = []
	with open(dataPath, 'r') as f:
		reader = csv.reader(f)
		for row in reader:
			numbers = [int(x) for x in row]
			Y.append(numbers[0])
			X.append(numbers[1:])
	return (np.stack(X).astype('float32'), np.stack(Y))

def getNetwork(input_shape, d1, d2, d3, output_shape, lr, act, d4=None):
	model = Sequential()
	model.add(Dense(d1, input_shape=input_shape))
	model.add(Activation(act))
	model.add(Dense(d2))
	model.add(Activation(act))
	model.add(Dense(d3))
	model.add(Activation(act))
	if d4:
		model.add(Dense(d4))
		model.add(Activation(act))
	model.add(Dense(output_shape))
	model.add(Activation('softmax'))
	model.compile(loss=keras.losses.categorical_crossentropy,
		optimizer=Adadelta(lr),
		metrics=['accuracy'])
	return model

def getSmallNetwork(input_shape, d1, d2, output_shape, lr):
	model = Sequential()
	model.add(Dense(d1, input_shape=input_shape))
	model.add(Activation('relu'))
	model.add(Dense(d2))
	model.add(Activation('relu'))
	model.add(Dense(output_shape))
	model.add(Activation('softmax'))
	model.compile(loss=keras.losses.categorical_crossentropy,
		optimizer=Adadelta(lr))
	return model

if __name__ == "__main__":
	# Get data
	(X_train, Y_train) = loadData('./emnist/emnist-balanced-train.csv')
	(X_test, Y_test) = loadData('./emnist/emnist-balanced-test.csv')
	X_train /= 255.
	X_test /= 255.
	Y_train = to_categorical(Y_train, 47)
	Y_test = to_categorical(Y_test, 47)
	# Part 0
	#m0 = getNetwork((784,), 256, 128, 64, 47, 0.1, 'linear')
	#m0.fit(X_train, Y_train, 32, epochs=100)
	#print(m0.evaluate(X_test, Y_test, 64)[1])
	# Part 1
	#LRs = [0.2, 0.1, 0.001]
	#accs = []
	#for lr in LRs:
	#	m1 = getNetwork((784,), 256, 128, 64, 47, lr, 'linear')
	#	m1.fit(X_train, Y_train, 32, epochs=100)
	#	accs.append(m1.evaluate(X_test, Y_test, 64)[1])
	#print(accs)
	# Part 2
	#m2 = getNetwork((784,), 512, 256, 128, 47, 0.1, 'linear')
	#m2.fit(X_train, Y_train, 32, epochs=100)
	#print(m2.evaluate(X_test, Y_test, 64)[1])
	#exit()
	# Part 3
	#m3 = getNetwork((784,), 256, 128, 64, 47, 0.1, 'linear', 128)
	#m3.fit(X_train, Y_train, 32, epochs=100)
	#print(m3.evaluate(X_test, Y_test, 64)[1])
	# Part 4
	m4 = getNetwork((784,), 256, 128, 64, 47, 0.1, 'linear')
	m4.fit(X_train, Y_train, 32, epochs=100, callbacks=[TensorBoard('./sml')])
	print(m4.evaluate(X_test, Y_test, 64)[1])
	# Part 5
	#m5 = getNetwork((784,), 256, 128, 64, 47, 0.1, 'relu')
	#m5.fit(X_train, Y_train, 32, epochs=15)
	#print(m5.evaluate(X_test, Y_test, 64)[1])
	# Part 6
	# Use best network so far, compress into 2 layer network using distillation
	#m6 = getSmallNetwork((784,), 256, 128, 47, 0.1)
	#m6.fit(X_train, Y_train, 32, epochs=20)
	#print(m6.evaluate(X_test, Y_test, 64))


from Call import Call
from Simulator import Simulator
from Calls import Calls
from Elevator import Elevator
from Building import Building
import sys
import json
import csv
import numpy as np

#
def init_elev_list():
    elev_l = []
    # initialize elevator list
    for current in elev_dict:
        e = Elevator(int(current['_id']), float(current['_speed']), int(current['_minFloor']),
                     int(current['_maxFloor']), float(current['_closeTime']), float(current['_openTime']),
                     float(current['_startTime']), float(current['_stopTime']))
        elev_l.append(e)
    return elev_l

if __name__ == "__main__":
    run = True
    args = sys.argv
    if len(args) == 3 or run:
        b_name = args[0]
        c_name = args[1]
        out_name = args[2]

        # Building.JSON
        b_string = open(b_name, )
        b_data = json.load(b_string)  # get data from json
        elev_dict = b_data['_elevators']  # convert elev dict to an elev list
        elev_list = init_elev_list()
        b = Building(b_data['_minFloor'], b_data['_maxFloor'], elev_list)  # init building

        # Calls.CSV
        with open(c_name, newline='') as f:
            reader = csv.reader(f)
            calls_data = list(reader)
        # get list of calls from CSV
        calls_list = []
        for i in range(len(calls_data)):
            # define each new call (start time, src, destination)
            c = Call(float(calls_data[i][1]), int(calls_data[i][2]), int(calls_data[i][3]))
            calls_list.append(c)

        # let end time = time the last call received *2
        last = calls_list.pop()
        end_time = last.time_called * 2
        calls_list.append(last)

        # initialize simulator and run
        sim = Simulator(building=b, elevators=elev_list, calls=calls_list, end_time=end_time)
        sim.run()
        # updated list of calls

        # # get output.CSV
        # with open('out.csv', 'w', encoding='UTF8', newline='') as csvfile:
        #     writer = csv.writer(csvfile)
        #     writer.writerow(sim.calls)

        np.savetxt('out.csv', [p for p in zip(sim.calls)], delimiter=',', fmt='%r')


import cvxpy as cp
import numpy as np

import sympy
from sympy import var
from sympy.matrices import Matrix, eye, zeros, ones, diag, GramSchmidt

var('e1 e2 g p1 p2 p3')

M = Matrix([[0, 0, -1], [1, 0, 0], [0, 1, 0]])

P = Matrix([[0, 1 / 2], [1 / 2, 0]])
S = Matrix([[0, 1 / 2], [1 / 2, -e1]])
SP2 = Matrix([[0, 1 / 2], [1 / 2, -e2]])
L = Matrix([[g, 0], [0, -1]])

supp = {
    "P": P,
    "S": S,
    "L": L
}

order = 'LLS'

def xp(order):
    x1 = Matrix.diag([p1 * supp[order[0]][0, 0], p2 * supp[order[1]][0, 0], p3 * supp[order[2]][0, 0]])
    x2 = Matrix.diag([p1 * supp[order[0]][0, 1], p2 * supp[order[1]][0, 1], p3 * supp[order[2]][0, 1]])
    x3 = Matrix.diag([p1 * supp[order[0]][1, 0], p2 * supp[order[1]][1, 0], p3 * supp[order[2]][1, 0]])
    x4 = Matrix.diag([p1 * supp[order[0]][1, 1], p2 * supp[order[1]][1, 1], p3 * supp[order[2]][1, 1]])
    Xp = Matrix([[x1, x2], [x3, x4]])
    mc = Matrix([[M], [Matrix.eye(3)]])
    Xp = mc.transpose() @ Xp @ mc
    return Xp

Xp = xp(order)

p1 = cp.Variable(1)
p2 = cp.Variable(1)
p3 = cp.Variable(1)
v1 = 5
v2 = 6
h = 2

Xp = Xp.subs([(e1, v1), (e2, v2), (g, h), (p1, p1), (p2, p2), (p3, p3)])

#
# Xp2=Xp.subs(p1,3)
#
#
# cp.bmat([[Xp[0, :]], [Xp[1, :]], [Xp[2, :]]])


# -*- coding: utf-8 -*-
from setuptools.command import build_ext
from distutils.unixccompiler import UnixCCompiler

__author__ = 'Jon Nappi'
__all__ = ['GolangBuildExt', 'GolangCompiler']

class GolangBuildExt(build_ext.build_ext):
    """Custom extension builder that sets builds extensions using the
    :class:`GolangCompiler` as the compiler class
    """

    def initialize_options(self):
        super(GolangBuildExt, self).initialize_options()
        self.compiler = GolangCompiler

class GolangCompiler(UnixCCompiler):
    """A Custom Compiler class type detailing how to use gccgo to compile go
    code into a C binary that Python can consume
    """

    compiler_type = 'go'

    # These are used by CCompiler in two places: the constructor sets
    # instance attributes 'preprocessor', 'compiler', etc. from them, and
    # 'set_executable()' allows any of these to be set.  The defaults here
    # are pretty generic; they will probably have to be set by an outsider
    # (eg. using information discovered by the sysconfig about building
    # Python extensions).
    executables = {
        'preprocessor': None,
        'compiler': ['gccgo'],
        'compiler_so': ['gccgo'],
        'compiler_cxx': None,
        'linker_so': ['gccgo'],
        'linker_exe': None,
        'archiver': ['ar', '-cr'],
        'ranlib': None,
    }

    # `language_map` is used to detect a source file or Extension target
    # language, checking source filenames. language_order is used to detect
    # the language precedence, when deciding what language to use when mixing
    # source types. For example, if some extension has two files with ".c"
    # extension, and one with ".cpp", it is still linked as c++. However, given
    # that this Compiler is only concerned with compiling .go files we don't
    # need to worry about compilation precendence
    language_map = {'.go': 'go'}
    language_order = ['go']
    src_extensions = ['.go']

    def __init__(self, verbose=0, dry_run=0, force=0):
        """Custom compiler init override implicitly sets the required base
        gccgo include directories
        """
        super(GolangCompiler, self).__init__(verbose, dry_run, force)
        self.include_dirs = ['/usr/lib/gccgo', '/usr/local/lib/gccgo']

    def library_dir_option(self, dir):
        """Return the compiler option to add 'dir' to the list of libraries
        linked into the shared library or executable.
        """
        return ''.join(['-L', dir])

    def library_option(self, lib):
        """Return the compiler option to add 'dir' to the list of libraries
        linked into the shared library or executable.
        """
        return ''.join(['-l', dir])


import functools

from absl.testing import absltest
import chex
import haiku as hk
import jax
import jax.numpy as jnp
from jax_privacy.src.dp_sgd import grad_clipping
from jax_privacy.src.dp_sgd import typing
import optax

NUM_SAMPLES = 4
NUM_CLASSES = 7
INPUT_SHAPE = (12, 12, 3)
MAX_NORM = 1e-4

def model_fn(inputs, num_classes):
  """Mini ConvNet."""
  out = hk.Conv2D(4, 3)(inputs)
  out = jax.nn.relu(out)
  out = hk.Conv2D(4, 3)(out)
  out = jax.nn.relu(out)
  out = jnp.mean(out, axis=[1, 2])
  out = hk.Linear(num_classes)(out)
  return out

def grad_clipped_per_sample_naive(forward_fn, clipping_norm):
  """Naive implementation for computing gradients clipped per-example."""
  grad_fn = jax.grad(forward_fn, has_aux=True)

  def accumulate(tree_acc, tree_new, coeff):
    return jax.tree_util.tree_map(
        lambda leaf_acc, leaf_new: leaf_acc + leaf_new * coeff,
        tree_acc,
        tree_new,
    )

  def clipped_grad_fn(params, network_state, rng_per_example, inputs):
    loss, (network_state, metrics) = forward_fn(
        params, network_state, rng_per_example, inputs)
    images, labels = inputs
    batch_size = len(images)
    grads = jax.tree_util.tree_map(jnp.zeros_like, params)
    grad_norms = []

    # compute one clipped gradient at a time
    for i in range(batch_size):
      # expand image: function expects a batch dimension
      input_i = (jnp.expand_dims(images[i], 0), labels[i])
      grad_i, unused_aux = grad_fn(
          params, network_state, rng_per_example, input_i)

      norm_grad_i = jnp.sqrt(
          sum(jnp.sum(x**2) for x in jax.tree_util.tree_leaves(grad_i)))

      # multiplicative factor equivalent to clipping norm
      coeff = jnp.minimum(1, clipping_norm / norm_grad_i) / batch_size

      # normalize by batch_size and accumulate
      grads = accumulate(grads, grad_i, coeff)
      grad_norms.append(optax.global_norm(grad_i))

    metrics = metrics.replace(
        per_example={
            'grad_norm': jnp.array(grad_norms),
            **metrics.per_example,
        }
    )
    return (loss, (network_state, metrics)), grads

  return clipped_grad_fn

class TestClippedGradients(chex.TestCase):
  """Check numerically that gradients are correctly clipped."""

  def setUp(self):
    super().setUp()

    rng_seq = hk.PRNGSequence(0)

    images = jax.random.normal(
        next(rng_seq), shape=(NUM_SAMPLES,) + INPUT_SHAPE)
    labels = jax.random.randint(
        next(rng_seq), shape=[NUM_SAMPLES], minval=0, maxval=NUM_CLASSES)
    labels_one_hot = hk.one_hot(labels, NUM_CLASSES)

    self.net = hk.transform_with_state(
        functools.partial(model_fn, num_classes=NUM_CLASSES))
    self.params, self.network_state = self.net.init(next(rng_seq), images)

    self.inputs = (images, labels_one_hot)
    self.rng_per_batch = next(rng_seq)
    self.rng_per_example = jax.random.fold_in(self.rng_per_batch, 1)

    self.tol = {'rtol': 1e-6, 'atol': 1e-6}

  def forward(self, params, state, rng_per_example, inputs):
    images, labels = inputs
    logits, state = self.net.apply(params, state, rng_per_example, images)
    loss_vector = optax.softmax_cross_entropy(logits, labels)
    metrics = typing.Metrics(
        per_example={'loss': loss_vector, 'a': jnp.ones([images.shape[0]])},
        scalars_avg={'loss': jnp.mean(loss_vector), 'b': jnp.ones([])},
    )
    return jnp.mean(loss_vector), (state, metrics)

  def forward_per_sample(self, params, state, rng_per_example, inputs):
    images, labels = inputs
    logits, state = self.net.apply(params, state, rng_per_example, images)
    loss_vector = optax.softmax_cross_entropy(logits, labels)
    metrics = typing.Metrics(
        per_example={'loss': loss_vector, 'a': jnp.ones([images.shape[0]])},
        scalars_avg={'loss': jnp.mean(loss_vector), 'b': jnp.ones([])},
    )
    return jnp.mean(loss_vector), (state, metrics)

  @chex.variants(with_jit=True, without_jit=True)
  def test_clipped_gradients(self):
    value_and_grad_fn = jax.value_and_grad(self.forward, has_aux=True)
    clipping_fn = grad_clipping.global_clipping(clipping_norm=MAX_NORM)

    grad_fn_1 = grad_clipping.value_and_clipped_grad_vectorized(
        value_and_grad_fn,
        clipping_fn,
    )

    grad_fn_2 = grad_clipping.value_and_clipped_grad_loop(
        value_and_grad_fn,
        clipping_fn,
    )

    grad_fn_3 = grad_clipped_per_sample_naive(
        self.forward,
        clipping_norm=MAX_NORM,
    )

    grad_fn_args = (
        self.params, self.network_state, self.rng_per_example, self.inputs)
    (loss_1, aux_1), grad_1 = self.variant(grad_fn_1)(*grad_fn_args)
    (loss_2, aux_2), grad_2 = self.variant(grad_fn_2)(*grad_fn_args)
    (loss_3, aux_3), grad_3 = self.variant(grad_fn_3)(*grad_fn_args)

    chex.assert_trees_all_close(loss_1, loss_2, loss_3, **self.tol)
    chex.assert_trees_all_close(aux_1, aux_2, aux_3, **self.tol)
    chex.assert_trees_all_close(grad_1, grad_2, grad_3, **self.tol)

  @chex.variants(with_jit=True, without_jit=True)
  def test_gradients_vectorized_and_loop_match_using_batch_rng(self):
    value_and_grad_fn = jax.value_and_grad(self.forward, has_aux=True)
    clipping_fn = lambda grads: (grads, optax.global_norm(grads))

    grad_fn_1 = grad_clipping.value_and_clipped_grad_vectorized(
        value_and_grad_fn,
        clipping_fn=clipping_fn,
    )

    grad_fn_2 = grad_clipping.value_and_clipped_grad_loop(
        value_and_grad_fn,
        clipping_fn=clipping_fn,
    )

    grad_fn_3 = grad_clipped_per_sample_naive(
        self.forward,
        clipping_norm=float('inf'),
    )

    grad_fn_args = (
        self.params, self.network_state, self.rng_per_example, self.inputs)
    (loss_1, aux_1), grad_1 = self.variant(grad_fn_1)(*grad_fn_args)
    (loss_2, aux_2), grad_2 = self.variant(grad_fn_2)(*grad_fn_args)
    (loss_3, aux_3), grad_3 = self.variant(grad_fn_3)(*grad_fn_args)

    chex.assert_trees_all_close(loss_1, loss_2, loss_3, **self.tol)
    chex.assert_trees_all_close(aux_1, aux_2, aux_3, **self.tol)
    chex.assert_trees_all_close(grad_1, grad_2, grad_3, **self.tol)

  @chex.variants(with_jit=True, without_jit=True)
  def test_gradients_vectorized_and_loop_match_using_per_sample_rng(self):
    clipping_fn = lambda grads: (grads, optax.global_norm(grads))

    grad_fn_1 = grad_clipping.value_and_clipped_grad_vectorized(
        jax.value_and_grad(self.forward_per_sample, has_aux=True),
        clipping_fn=clipping_fn,
    )

    grad_fn_2 = grad_clipping.value_and_clipped_grad_loop(
        jax.value_and_grad(self.forward_per_sample, has_aux=True),
        clipping_fn=clipping_fn,
    )

    grad_fn_args = (
        self.params, self.network_state, self.rng_per_example, self.inputs)
    (loss_1, aux_1), grad_1 = self.variant(grad_fn_1)(*grad_fn_args)
    (loss_2, aux_2), grad_2 = self.variant(grad_fn_2)(*grad_fn_args)

    chex.assert_trees_all_close(loss_1, loss_2, **self.tol)
    chex.assert_trees_all_close(aux_1, aux_2, **self.tol)
    chex.assert_trees_all_close(grad_1, grad_2, **self.tol)

@hk.transform_with_state
def simple_net(x):
  """A simple function that computes L = 3 * x + random noise."""
  key = hk.next_rng_key()
  noise = jax.random.normal(key, shape=x.shape)
  return jnp.mean(3 * x + noise), noise

class TestClippedGradientsPerBatchPerSampleRNG(chex.TestCase):
  """Check per-batch rng and per-sample rng are handled correctly."""

  def setUp(self):
    super().setUp()
    rng_seq = hk.PRNGSequence(0)
    self.inputs = jax.random.normal(next(rng_seq), shape=(NUM_SAMPLES, 10))
    self.params, self.state = simple_net.init(next(rng_seq), self.inputs)
    self.rng_per_batch = next(rng_seq)
    self.rng_per_example = jax.random.fold_in(self.rng_per_batch, 1)

    self.grad_fn_args = (self.params, self.state,
                         self.rng_per_example, self.inputs)
    self.no_clip = lambda grads: (grads, optax.global_norm(grads))

  @chex.variants(with_jit=True, without_jit=True)
  def test_per_sample_rng_produces_different_random_numbers(self):

    @functools.partial(jax.value_and_grad, has_aux=True)
    def value_and_grad_fn(params, state, rng_per_example, inputs):
      (loss, noise), state = simple_net.apply(
          params, state, rng_per_example, inputs)
      metrics = typing.Metrics(
          per_example={'noise': noise},
      )
      return loss, (state, metrics)

    with self.subTest('vectorized'):
      grad_fn = grad_clipping.value_and_clipped_grad_vectorized(
          value_and_grad_fn, clipping_fn=self.no_clip)
      (_, (_, metrics)), _ = self.variant(grad_fn)(*self.grad_fn_args)

      # Noise should be different across all samples.
      for i in range(1, NUM_SAMPLES):
        self.assertTrue(
            jnp.any(metrics.per_example['noise'][0]
                    != metrics.per_example['noise'][i]))

    with self.subTest('loop'):
      grad_fn = grad_clipping.value_and_clipped_grad_loop(
          value_and_grad_fn, clipping_fn=self.no_clip)
      (_, (_, metrics)), _ = self.variant(grad_fn)(*self.grad_fn_args)

      # Noise should be different across all samples.
      for i in range(1, NUM_SAMPLES):
        self.assertTrue(
            jnp.any(metrics.per_example['noise'][0]
                    != metrics.per_example['noise'][i]))

if __name__ == '__main__':
  absltest.main()


"""© 2022 The Trustees of Columbia University in the City of New York.
This work may be reproduced, distributed, and otherwise exploited for
academic non-commercial purposes only, provided that it cites the original work:

    Stefan J. Hofmeister, Michael Hahn, and Daniel Wolf Savin, "Deriving instrumental point spread functions from partially occulted images,"
    Journal of the Optical Society of America A 39, 2153-2168 (2022), doi: 10.1364/JOSAA.471477.

To obtain a license to use this work for commercial purposes, please contact
Columbia Technology Ventures at techventures@columbia.edu <mailto:techventures@columbia.edu>.
"""

import numpy as np
import glob
import os
from inout import read_image, save_image
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
from matplotlib.colors import Normalize, LogNorm

#This is the top level routine for identifying the occulted pixels
def find_occulted_pixels(config):
    folder_run = config['general']['folder_run']
    method     = config['occultation_mask']['method']
    if 'add_distance' in config['occultation_mask']: add_distance = config['occultation_mask']['add_distance']
    else: add_distance = 0

    #if occultation masks have been predefined
    if method == 'predefined': files = glob.glob(folder_run + '/occultation_mask/*.npz')
    else:                      files = glob.glob(folder_run + '/deconvolved_image/*.npz')

    for file in files:
        #here, you can also define your own methods for finding the occulted pixels
        if method == 'predefined':
            #if the occultation mask has been provided by the user
            occultation_mask = convert_toimage(file, dtype = np.int8, save = False)
            img = read_image(folder_run + '/deconvolved_image/' + os.path.splitext(os.path.basename(file))[0] + '.npz')

        if method == 'threshold':
            #derive the occultation masks by an adaptive threshold technique
            img = read_image(file, dtype = np.float16)
            thr        = config['occultation_mask']['threshold']
            use_median = config['occultation_mask']['threshold_usemedian']
            occultation_mask = find_occulted_pixels_bythr(img, thr, use_median)

        if add_distance > 0:
            kernel_size = 2 * np.floor(add_distance).astype(np.int) + 1
            kx, ky = np.meshgrid(np.arange(kernel_size) - kernel_size//2, np.arange(kernel_size) - kernel_size//2)
            kr = kx**2 + ky**2
            kernel = np.zeros((kernel_size, kernel_size))
            kernel[kr <= add_distance**2] = 1
            mask_ignore = (occultation_mask == -1)
            occultation_mask = np.logical_not(occultation_mask)
            occultation_mask = ndimage.convolve(occultation_mask.astype(np.int8), kernel.astype(np.int8), mode = 'nearest')
            occultation_mask = np.logical_not(occultation_mask).astype(np.int8)
            occultation_mask[mask_ignore] = -1

        #plot the occultation mask, one time solely the occultation mask and one time overlayed to the image
        filename_figure = os.path.splitext(file)[0] + '.jpg'
        fig, ax = plt.subplots(1, 2, figsize = (12, 5))
        ax1 = ax[0]
        divider = make_axes_locatable(ax1)
        cax1 = divider.append_axes('right', size='5%', pad=0.05)
        im1 = ax1.imshow(occultation_mask, norm = Normalize(vmin = -1, vmax = 1, clip = True), interpolation = 'nearest', cmap = 'viridis')
        fig.colorbar(im1, cax=cax1, orientation='vertical', ticks=[-1, 0, 1])
        ax2 = ax[1]
        divider = make_axes_locatable(ax2)
        cax2 = divider.append_axes('right', size='5%', pad=0.05)
        img_occ_marked = np.copy(img)
        img_occ_marked[occultation_mask == 1] = 10000
        im2 = ax2.imshow(img_occ_marked,  norm=LogNorm(vmin=1, vmax=10000, clip = True), interpolation = 'nearest', cmap = 'viridis')
        fig.colorbar(im2, cax=cax2, orientation='vertical')
        plt.tight_layout()
        plt.savefig(filename_figure, dpi = 600)
        plt.close(fig)
        #finally, save the occultation mask
        save_image(occultation_mask, folder_run + '/occultation_mask/' + os.path.basename(file), plot = False, dtype = np.int8, keys = 'img')

#In this section, you can put your own algorithms to identify occulted pixels. The function should use the (psf deconvolved) image "img" as input, and return a numpy array with a zero background and the pixels identified as occulted set to one.

def find_occulted_pixels_bythr(img, thr, use_median = True):
    #define if using the median or mean of the illuminated portion of the image as references
    if use_median: avg_function = np.median
    else: avg_function = np.mean

    #define a mask containing the occulted pixels
    occulted_pixels = np.zeros(img.shape)

    #derive the mean/median of the image and set the occulted pixels as all pixels smaller than threshold * mean/median. Rederive the mean/median only from the not-occulted pixels, find the occulted pixels, and iterate until the result converges.
    img_mean_last = 0.
    for i in range(100): #use 100 as the max number of iterations. Usually, <10 iterations are sufficient for convergence
        img_mean = avg_function(img[occulted_pixels == 0])
        occulted_pixels = img < (img_mean * thr)
        if img_mean == img_mean_last:
            break
        else:
            img_mean_last = img_mean
    return np.array(occulted_pixels)


### 052830 -- doing PCA on temporal neural patterns
### can I think be analogized to a PSTH (as the first component)
### both to examine the PCs, and to see how many are needed

import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

from sklearn.decomposition import PCA
from analysis import FIG_DIR
from analysis.data_utils import get_neural_data, spike_counts, \
        trial_average

# set up data -- a few choices here
DATASET = 'tang' # want lots of trials for the trial-averaged part
CORR_THRESHOLD = 0.8 # doesn't hurt to be selective -- want real neurons
SORT = 'batch' # still some concerns about the Smith sort
WINDOW_SIZE = 20
START = 500
STOP = 1100 # why not get the response to the image disappearing?
num_bins = len(range(START, STOP, WINDOW_SIZE))

data = get_neural_data(DATASET, CORR_THRESHOLD, sort=SORT)
# will analyze both trial-averaged and trial-to-trial dynamics
# could be interesting if they're different (?)
avg_data = trial_average(data)

# think the PCA analysis means smoothing isn't really necessary
# since we have lots of data points
data = [spike_counts(data, start, start+WINDOW_SIZE) for
        start in range(START, STOP, WINDOW_SIZE)]
avg_data = [spike_counts(avg_data, start, start+WINDOW_SIZE) for
        start in range(START, STOP, WINDOW_SIZE)]

# at this point
# data is a T-lengh list of C-lenth lists of T_c x N tensors
# avg_data is a T-length list of CxN tensors
# want to convert them to (whatever) x T
data = np.array([np.concatenate(time).flatten() for time in data]).transpose(1, 0)
avg_data = np.array([time.flatten() for time in avg_data]).transpose(1, 0)

# now can do PCA on each of these
pca = PCA().fit(data)
avg_pca = PCA().fit(avg_data)

pca_ninety = np.argmax(np.cumsum(pca.explained_variance_ratio_) > 0.9)
avg_pca_ninety = np.argmax(np.cumsum(avg_pca.explained_variance_ratio_) > 0.9)

print(f'{pca_ninety} of {num_bins} PCs are needed to explain 90% of trial PSTH variance')
print(f'{avg_pca_ninety} of {num_bins} PCs are need to explain 90% of average PSTH variance')

# more important is looking at the PCs themselves -- interpretable as PSTHs
# (rather components that form PSTHs, but the same graphically)
# plot out the important ones

for i in range(pca_ninety+1):
    if i == 0:
        this_pc = pca.mean_ * (1000 / WINDOW_SIZE)
        ratio = 0
    else:
        this_pc = pca.components_[i-1] * (1000 / WINDOW_SIZE) # convert to Hz
        ratio = np.round(pca.explained_variance_ratio_[i-1], 2) * 100

    plt.figure()
    plt.plot(this_pc, color='black', linewidth=3)
    plt.xlabel(f'{WINDOW_SIZE}ms time bin')
    plt.ylabel('response change from mean (Hz)')
    plt.title(f'PC {i} of single-trial PSTHs, explaining {ratio}% var')
    plt.savefig(f'{FIG_DIR}trial_PSTH_pc{i:02}.png')

for i in range(avg_pca_ninety):
    if i == 0:
        this_pc = avg_pca.mean_ * (1000 / WINDOW_SIZE)
        ratio = 0
    else:
        this_pc = avg_pca.components_[i-1] * (1000 / WINDOW_SIZE) # convert to Hz
        ratio = np.round(avg_pca.explained_variance_ratio_[i-1], 2) * 100

    plt.figure()
    plt.plot(this_pc, color='black', linewidth=3)
    plt.xlabel(f'{WINDOW_SIZE}ms time bin')
    plt.ylabel('response change from mean (Hz)')
    plt.title(f'PC {i} of trial-averaged PSTHs, explaining {ratio}% var')
    plt.savefig(f'{FIG_DIR}avg_PSTH_pc{i:02}.png')


import copy

import pygame

from sim_utils.config import LANE_LIGHT_LOCATION, LANE_LENGTH, FACTOR_SPEED, SAFETY_DISTANCE, EMERGENCY_RED_MAX_SPEED, \
    METHOD
from sim_utils.utils import load_image, get_screen_center, get_lane_points, stopping_position, get_rotation, \
    acceleration_time

class Vehicle(pygame.sprite.Sprite):
    def __init__(self, name: str, speed: int, max_speed: int,
                 acceleration: int, braking: int, turn_rate: int, scale: tuple,
                 direction: str,  dataStorage, next_lane, id_: int = -1, time_waited: int = 0, debug=False):
        self.dataStorage = dataStorage
        pygame.sprite.Sprite.__init__(self)

        self.load_car_image(direction, name, scale)

        # Font for debugging purposes
        self.font = pygame.font.SysFont("monospace", 14)

        # Initially start the car at the start of the lane
        self.position = 0.0

        self.direction = direction
        self.next_lane = next_lane
        self.id = id_
        self.name = name
        self.scale = scale
        # TODO: better solution to determine length/width
        self.width = min(self.rect.height, self.rect.width)
        self.length = max(self.rect.height, self.rect.width)
        self.rect = pygame.Rect(-100, -100, self.rect.width, self.rect.height)
        self.speed = speed
        self.max_speed = max_speed
        self.acceleration = acceleration
        self.turn_rate = turn_rate
        self.braking = braking
        self.turning = False
        self.turning_start = None
        self.reached_destination = False
        self.ticks = 0
        self.wait_time = time_waited

        self.debug = debug
        # TODO: debug
        self.queue_length = 0

        if self.next_lane and not self.direction_change(self.next_lane, self.direction):
            # No direction change, zero turn rate
            self.turn_rate = 0

    def update_cycle(self, lane, queue_length, previous_car, intersection_blocked):
        self.ticks += 1

        # If car is stopped in queue, increase wait time
        if lane.towards and self.speed == 0:
            self.wait_time += 1

        if self.turning and self.direction_change(self.next_lane, self.direction):
            self.speed = 0

        elif (previous_car is not None and
              (stopping_position(self.position, max(self.length, self.width), self.speed, self.braking) >
               stopping_position(previous_car.position, max(previous_car.length, previous_car.width),
                                 previous_car.speed, previous_car.braking) -
               max(previous_car.length, previous_car.width) / LANE_LENGTH - SAFETY_DISTANCE)):
            # Braking now avoids a collision with the car in front
            self.speed = max(0, self.speed - self.braking)

        elif lane.checklight() == 'green':
            # Light is green; accelerate
            self.speed = min(self.speed + self.acceleration, self.max_speed)

        elif self.passed_light():
            # Already passed the light; accelerate
            self.speed = min(self.speed + self.acceleration, self.max_speed)

        elif (stopping_position(self.position, max(self.length, self.width), self.speed, self.braking) >
              LANE_LIGHT_LOCATION - SAFETY_DISTANCE):
            # Braking now stops before the light
            self.speed = max(0, self.speed - self.braking)

        else:
            # No action required, accelerate
            self.speed = min(self.speed + self.acceleration, self.max_speed)

        if (lane.towards and
            intersection_blocked and
            LANE_LIGHT_LOCATION > self.position > (LANE_LIGHT_LOCATION - 0.05)):
            # Car is about to enter intersection but emergency is crossing; brake
            self.speed = max(0, self.speed - self.braking)

        if (previous_car is not None and
            previous_car.position >LANE_LIGHT_LOCATION and
            self.direction_change(previous_car.next_lane, previous_car.direction)):
            # Car in front is turning, wait at light
            self.speed = max(0, self.speed - self.braking)

        self.position += (self.speed * FACTOR_SPEED) / LANE_LENGTH

        # TODO: debug
        self.queue_length = queue_length

        keep_car = True
        if self.position > 1.0:
            if self.next_lane:
                # Car transitions to next lane
                if self.turning:
                    if self.ticks - self.turning_start > self.turn_rate:
                        # Car is done turning, remove car, add one to next lane
                        self.turning = False
                        keep_car = False
                        self.next_lane[0].addCar(
                            Vehicle(self.name, self.speed, self.max_speed,
                                    self.acceleration, self.braking, self.turn_rate, self.scale,
                                    self.next_lane[1], self.dataStorage, None,
                                    time_waited=self.wait_time, debug=self.debug)
                        )
                else:
                    # Start turning on the intersection
                    self.turning = True
                    self.turning_start = self.ticks
            else:
                # Car has reached destination
                self.dataStorage.add_destination(self.direction, self.ticks, self.wait_time)
                keep_car = False

        if not keep_car:
            # Remove sprite from group
            self.kill()
            return False
        return True

    def update_emergency(self, lane):
        self.ticks += 1

        if self.position < (LANE_LIGHT_LOCATION - 0.15):
            # Accelerate towards max speed
            self.speed = min(self.speed + self.acceleration, self.max_speed)
        elif lane.checklight() != 'green':
            # Slow down to maximum red traversing speed
            self.speed = max(EMERGENCY_RED_MAX_SPEED, self.speed - self.braking)
        else:
            # Accelerate
            self.speed = min(self.speed + self.acceleration, self.max_speed)

        self.position += (self.speed * FACTOR_SPEED) / LANE_LENGTH

        # TODO: Below is copied from update_cycle()
        keep_car = True
        if self.position > 1.0:
            if self.next_lane:
                # Car transitions to next lane
                if self.turning:
                    if self.ticks - self.turning_start > self.turn_rate:
                        # Car is done turning, remove car, add one to next lane
                        self.turning = False
                        keep_car = False
                        self.next_lane[0].add_emerg_vehicle(
                            Vehicle(self.name, self.speed, self.max_speed,
                                    self.acceleration, self.braking, self.turn_rate, self.scale,
                                    self.next_lane[1], self.dataStorage, None, debug=self.debug)
                        )
                else:
                    # Start turning on the intersection
                    self.turning = True
                    self.turning_start = self.ticks
            else:
                # Car has reached destination
                keep_car = False

        if not keep_car:
            # Remove sprite from group
            self.kill()
            return False
        return True

    def render(self, screen, lane, prev_car, side_parked=False):
        if side_parked:
            start, end = get_lane_points(lane, get_screen_center(), order_offset=1)
        else:
            start, end = get_lane_points(lane, get_screen_center(), center_line=True)

        if self.direction == 'E':
            self.rect.center = (start.x + self.position * LANE_LENGTH,
                                start.y)
        if self.direction == 'W':
            self.rect.center = (start.x - self.position * LANE_LENGTH,
                                start.y)
        if self.direction == 'N':
            self.rect.center = (start.x,
                                start.y - self.position * LANE_LENGTH)
        if self.direction == 'S':
            self.rect.center = (start.x,
                                start.y + self.position * LANE_LENGTH)
        if self.debug:
            # debug_output = str(round(self.position, 2))
            previous_id = -1
            if prev_car is not None:
                previous_id = prev_car.id
            debug_output = 'None'
            # debug_output = str(self.id) + ', ' + str(previous_id)
            # debug_output = str(self.direction)
            # debug_output = str(self.queue_length)
            if lane.light is not None:
                debug_output = str(round(lane.light.getGreentime(), 2))
            # debug_output = str(round(acceleration_time(1.0-self.position, self.speed, self.acceleration,
            #                                            self.max_speed), 2))
            # debug_output = str(lane.emergency_active)

            text = self.font.render(debug_output, 1, (255, 255, 255), (0, 0, 0))
            screen.blit(text, self.rect.bottomright)

    def passed_light(self):
        return self.position > LANE_LIGHT_LOCATION

    def rotate_car(self, angle_start, angle_end, portion):
        current_angle = portion * (angle_end - angle_start)
        self.image = pygame.transform.rotate(self.image, current_angle)
        self.rect = self.image.get_rect()

    def load_car_image(self, direction, vehicle_name, scale):
        self.image, self.rect = load_image(vehicle_name + '.png', -1)

        # Scale original image
        new_width = int(self.rect.width * scale[0])
        new_height = int(self.rect.height * scale[1])
        self.image = pygame.transform.scale(self.image, (new_width, new_height))

        # Rotate image according to direction
        rotation_angle = get_rotation(direction)
        self.image = pygame.transform.rotate(self.image, rotation_angle)
        self.rect = self.image.get_rect()

    @staticmethod
    def direction_change(next_lane, direction):
        if next_lane and next_lane[1] != direction:
            return True
        return False


"""Heroku dataclip event source"""

import csv
from datetime import datetime
import itertools
import json
import time
import requests
import dateutil.parser

from eventplayback import EventSource

class DataClipSource(EventSource):
    """A source of events, replayed from a Heroku dataclip.

    See dataclip.json.example for a sample config file."""

    def __init__(self, config_file):
        self._date_parser = dateutil.parser
        self._stats = {}
        self._buffer = []

        with open(config_file) as config_in:
            config = json.load(config_in)
            self.url = config['url']
            self._window_size = config['window_size']
            self._window_offset = config['window_offset']
            self._time_column = config['time_column']

    def get_date(self, event):
        return self._date_parser.parse(event[self._time_column]+'Z')

    @property
    def window_size(self):
        return self._window_size

    @property
    def window_offset(self):
        return self._window_offset

    def _read_clip(self):
        result = requests.get(self.url)

        if result.status_code == 200:
            return list(csv.DictReader(result.text.splitlines()))

        print("Error accessing data: {}".format(result))
        return []

    @property
    def stats(self):
        return self._stats

    def _merge_ordered_events(self, events1, events2):
        """Merge two lists of events, removing duplicates."""
        return [k for k, _ in itertools.groupby(sorted(
            events1 + events2, key=lambda e: e[self._time_column]))]

    def events(self, start, end):
        print("Window: {} and {}".format(datetime.fromtimestamp(start/1000),
                                         datetime.fromtimestamp(end/1000)))
        try:
            new_events = self._read_clip()
            unfiltered_events = self._merge_ordered_events(new_events, self._buffer)
            total_event_count = len(unfiltered_events)
            events = [event for event in unfiltered_events
                      if start <= (self.get_date(event).timestamp() * 1000) < end]
            events_earlier_count = len([event for event in unfiltered_events
                                        if (self.get_date(event).timestamp() * 1000) < start])
            events_later = [event for event in unfiltered_events
                            if (self.get_date(event).timestamp() * 1000) >= end]
            events_later_count = len(events_later)
            self._buffer = events_later
            self._stats['events_total'] = total_event_count
            self._stats['events_in_window'] = len(events)
            self._stats['events_before_window'] = events_earlier_count
            self._stats['events_after_window'] = events_later_count
            self._stats['events_from_dataclip'] = len(new_events)
            print("{:d}/{:d} events in window ({:d} earlier, {:d} later)".format(
                len(events), total_event_count, events_earlier_count, events_later_count))
            print("{:d}/{:d} events in window".format(len(events), total_event_count))
            self._stats['event_date_first'] = None
            self._stats['event_date_last'] = None
            if unfiltered_events:
                first = self.get_date(unfiltered_events[0])
                last = self.get_date(unfiltered_events[-1])
                self._stats['event_date_first'] = str(first)
                self._stats['event_date_last'] = str(last)
                print("First is {} last is {}".format(first, last))

            return events

        except requests.RequestException as err:
            print("Got an error polling the dataclip <{}>. Waiting a minute and retrying...".format(err))
            time.sleep(60)
        return []


import os
# import multiprocessing
import librosa
import soundfile as sf

# import soxbindings as sox

def preprocess(data, input_dir, dest_dir, target_sr=None, speed=None,
               overwrite=True):
    speed = speed or []
    speed.append(1)
    speed = list(set(speed))  # Make uniqe

    input_fname = os.path.join(input_dir,
                               data['input_relpath'],
                               data['input_fname'])

    # input_sr = sox.file_info.sample_rate(input_fname)
    f = open(input_fname, 'rb')
    file = sf.SoundFile(f)
    # metadata = soundfile.info(file, verbose=True)
    # file = audioread.rawread.RawAudioFile(input_fname)
    y, input_sr = librosa.load(file)
    target_sr = target_sr or input_sr

    os.makedirs(os.path.join(dest_dir, data['input_relpath']), exist_ok=True)

    output_dict = {}
    output_dict['transcript'] = data['transcript'].lower().strip()
    output_dict['files'] = []

    fname = os.path.splitext(data['input_fname'])[0]
    for s in speed:
        output_fname = fname + '{}.wav'.format('' if s==1 else '-{}'.format(s))
        output_fpath = os.path.join(dest_dir,
                                    data['input_relpath'],
                                    output_fname)

        # if not os.path.exists(output_fpath) or overwrite:
            # cbn = sox.Transformer().speed(factor=s).convert(target_sr)
            # cbn.build(input_fname, output_fpath)
        y_out = librosa.resample(y, orig_sr=input_sr, target_sr=target_sr)
        y_out = librosa.effects.time_stretch(y_out, rate=s)
        with open(output_fpath, 'wb') as wf:
            sf.write(wf, y_out, target_sr)

        # file_info = sox.file_info.info(output_fpath)
        # file_info = {
        #     "name": metadata.name,
        #     "samplerate": metadata.samplerate,
        #     "channels": metadata.channels,
        #     "frames": metadata.frames,
        #     "duration": metadata._duration_str,
        #     "format": f"{metadata.format_info} ({metadata.format})",
        #     "subtype": f"{metadata.subtype_info} ({metadata.subtype})",
        # }
        file_info = {
            "name": output_fname,
            "samplerate": target_sr,
            "channels": y_out.shape[0] if len(y_out.shape) > 1 else 1,
            "frames": y_out.shape[1] if len(y_out.shape) > 1 else y_out.shape[0],
            "duration": (y_out.shape[1] / target_sr) if len(y_out.shape) > 1 else (y_out.shape[0] / target_sr),
        }
        file_info['fname'] = os.path.join(os.path.basename(dest_dir),
                                          data['input_relpath'],
                                          output_fname)
        file_info['speed'] = s
        output_dict['files'].append(file_info)

        # if s == 1:
            # file_info = sox.file_info.info(output_fpath)
            # output_dict['original_duration'] = file_info['duration']
            # output_dict['original_num_samples'] = file_info['num_samples']

    f.close()

    return output_dict

# def parallel_preprocess(dataset, input_dir, dest_dir, target_sr, speed, overwrite, parallel):
#     with multiprocessing.Pool(parallel) as p:
#         func = functools.partial(preprocess,
#             input_dir=input_dir, dest_dir=dest_dir,
#             target_sr=target_sr, speed=speed, overwrite=overwrite)
#         dataset = list(tqdm(p.imap(func, dataset), total=len(dataset)))
#         return dataset


import sys
import os

sys.path.append("..")
import json

def split_json(file_path, chunk_size):
    with open(file_path, "r", encoding="utf-8") as file:
        data = json.load(file)  # Assuming it's a JSON array

        for i in range(0, len(data), chunk_size):
            with open(f"chunk_{i}.json", "w", encoding="utf-8") as chunk_file:
                json.dump(
                    data[i : i + chunk_size], chunk_file, ensure_ascii=False, indent=4
                )

def get_true_captions(file_path):
    with open(file_path, "r", encoding="utf-8") as file:
        data = json.load(file)

        true_captions = [item["true_caption"] for item in data]

        with open("true_captions.json", "w", encoding="utf-8") as new_file:
            json.dump(true_captions, new_file, ensure_ascii=False, indent=4)

def get_false_captions(file_path):
    with open(file_path, "r", encoding="utf-8") as file:
        data = json.load(file)

        true_captions = [item["false_caption"] for item in data]

        with open("false_captions.json", "w", encoding="utf-8") as new_file:
            json.dump(true_captions, new_file, ensure_ascii=False, indent=4)

def replace_captions(original, new):
    with open(original, "r", encoding="utf-8") as file:
        original_data = json.load(file)

    # Step 1: Read the modified false captions file
    with open(new, "r", encoding="utf-8") as file:
        modified_captions = json.load(file)

    # Step 2: Replace 'false_caption' in the original data
    for i, item in enumerate(original_data):
        if i < len(modified_captions):
            item["false_caption"] = modified_captions[i]
        else:
            break  # Break if there are more items in original_data than in modified_captions

    # Step 3: Write the updated data back to a new file
    with open("updated_file.json", "w", encoding="utf-8") as file:
        json.dump(original_data, file, ensure_ascii=False, indent=4)

def filter_json_strings(input_file_path, output_file_path, num_items):
    """
    Reads a JSON file containing a list of strings, filters a specified percentage of the strings,
    and writes the filtered list to a new JSON file.

    :param input_file_path: Path to the input JSON file.
    :param output_file_path: Path to the output JSON file.
    :param percentage: The percentage of strings to include in the output file.
    :return: None
    """
    try:
        # Reading data from the input file
        with open(input_file_path, "r") as file:
            data = json.load(file)

        if not isinstance(data, list):
            raise ValueError("JSON file does not contain a list")

        # Calculating the number of items to include
        # num_items = int(len(data) * (percentage / 100))
        num_items = num_items

        # Selecting a subset of strings
        filtered_data = data[:num_items]

        # Writing the filtered data to the output file
        with open(output_file_path, "w") as file:
            json.dump(filtered_data, file)

        return "Filtered JSON file created successfully."
    except Exception as e:
        return f"An error occurred: {e}"

# Example usage:
# filter_json_strings('path/to/input.json', 'path/to/output.json', 50)

def main():
    root_dir = "C:/Users/ewang/OneDrive/Desktop/Fall 2023/CompVLMs/vision-language-models-are-bows/data2"
    annotation_file = os.path.join(root_dir, "visual_genome_attribution.json")
    # get_true_captions(annotation_file)
    # split_json("your_large_file.json", 1000)  # Adjust chunk_size as needed
    # replace_captions(annotation_file, "replace-att-modified-0.json")
    # get_false_captions(annotation_file)

    filter_json_strings("rel-original-true.json", "rel-original-true-20k.json", 20000)

if __name__ == "__main__":
    main()


import os
from random import randint
from flask import Flask, redirect, render_template, flash, request
# from app import clear_app_cache, invalidate_key_in_app_cache, put_key_in_app_cache, refresh_app_cache_config
from app import *
from lib import calc_statistics, delete_images_data, get_cache_config, get_capacity, get_last_10_min_stat, get_path, get_size, key_exist, save_mem_config

flask_app = Flask(__name__)

UPLOAD_FOLDER = './static/uploads'

flask_app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
flask_app.config['SESSION_TYPE'] = 'filesystem'
flask_app.config.update(SECRET_KEY=os.urandom(24))

@flask_app.route("/")
def index():
    return redirect('/app_config',code=302)

@flask_app.route("/app_config", methods=['GET'])
def cache_config_get():
    # print("Policies: ", get_policies())
    data = get_cache_config()
    # data['random1'] = [randint(1,9999) for x in range(30)]
    # data['random2'] = [randint(1,9999) for x in range(30)]
    # data['random3'] = [randint(1,9999) for x in range(30)]
    # data['random4'] = [randint(1,9999) for x in range(30)]
    # data['random5'] = [randint(1,9999) for x in range(30)]
    return render_template('cache_config.html', data = data)

@flask_app.route("/app_config", methods=['POST'])
def cache_config_post():
    data = get_cache_config()
    if 'policy' not in request.form:
        flash('No Policy Added','error')
        return render_template('cache_config.html', data = data)

    if 'capacity' not in request.form: capacity = get_capacity()
    else: capacity = request.form['capacity']

    policy = request.form['policy']
    mem_policy = request.form['mem_policy']

    if save_mem_config(policy,mem_policy,capacity): flash("Settings Saved",'msg')
    else: flash("Could not save setting.",'error')

    refresh_app_cache_config()
    flash("Cache Refreshed",'msg')

    data['capacity'] = capacity
    data['current_policy_id'] = int(policy)
    data['current_memcache_policy_id'] = int(mem_policy)
    # print("Data: ",data)
    return render_template('cache_config.html', data = data)

@flask_app.route("/statictics", methods=['GET'])
def statistics_get():
    data = {}
    log = get_last_10_min_stat()
    calculated_stat = calc_statistics(get_last_10_min_stat()) # tuple(requests, hit, miss)
    requests = calculated_stat[0]
    data['served_requests'] = requests
    data['hit_rate'] = float(calculated_stat[1] / requests * 100) if requests > 0 else 0
    data['miss_rate'] = float(calculated_stat[2] / requests * 100) if requests > 0 else 0
    data['log'] = log

    return render_template('statistics.html', data=data)

@flask_app.route("/clear_cache", methods=['GET'])
def clear_cache():
    data = get_cache_config()
    # app.clear_app_cache()
    cache.clear()
    flash('Cache Cleared Successfully.', 'msg')
    return render_template('cache_config.html',data = data)

@flask_app.route("/clear_app_data", methods=['GET'])
def clear_app_data():
    data = get_cache_config()
    clear_app_cache()
    flash('Cache Cleared Successfully.', 'msg')

    if delete_images_data(): flash('App Data Cleared Successfully.', 'msg')
    else: flash('Can not Clear App Data :(', 'error')

    return render_template('cache_config.html',data = data)

@flask_app.route("/invalidate_key", methods=["POST"])
def invalidate_key():
    data = get_cache_config()
    if 'key' not in request.form:
        flash('No Key Added','error')
        return render_template('cache_config.html', data = data)
    # print("--------------degub Lineee----------------")

    key = request.form['key']

    if key == '':
        flash('No Key Added', 'error')
        return render_template('cache_config.html', data = data)

    if not key_exist(key):
        flash('Unexisted Key','error')
        return render_template('cache_config.html', data = data)

    invalidate_key_in_app_cache(key)
    flash('Key Invalidated Successfully', 'msg')
    return render_template('cache_config.html', data = data)

@flask_app.route("/put_key", methods=["POST"])
def insert_into_cache():
    data = get_cache_config()
    if 'key' not in request.form:
        flash('No Key Added','error')
        return render_template('cache_config.html', data = data)

    key = request.form['key']

    if key == '':
        flash('No Key Added', 'error')
        return render_template('cache_config.html', data = data)

    if not key_exist(key):
        flash('Not Used Key', 'error')
        return render_template('cache_config.html', data = data)

    path = get_path(key)
    size = float(get_size(key))
    cache.invalidate_key(key)

    saved = cache.put(key,path,size)

    if not saved:
        flash('Image Size > Cache Cpacity, Can not Save', 'error')
        return render_template('cache_config.html', data = data)

    flash('Element Inserted Into Memory Cache Successfully', 'msg')
    return render_template('cache_config.html', data = data)

if __name__ == "__main__":
    flask_app.run(host='0.0.0.0', port='9999', debug=True)


#! /usr/bin/env python 3

import numpy as np
import pandas as pd # data analysis package
import matplotlib.pyplot as plt

# filepath of csv data
FILEPATH = '/Users/gerald/Documents/TA-ing/MCHE 474/Me'
FILE_NAME = 'MCHE474-001-202240 Grades.csv'

df = pd.read_csv(f'{FILEPATH}/{FILE_NAME}', index_col='Last name')

hist_array = df['Test 2 (Real)'].to_numpy()

# clip the outliers to be within the bin ranges
hist_array = np.clip(hist_array, 0, 100)

# define bin ranges
bin_ranges = np.arange(0,101,10)

#############################################################
fig = plt.figure(figsize=(6,4))
ax = plt.gca()
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)

# Change the axis units font
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
plt.setp(ax.get_xmajorticklabels(),fontsize=18)

ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')

ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')

# Turn on the plot grid and set appropriate linestyle and color
ax.grid(True,linestyle=':', color='0.75')
ax.set_axisbelow(True)

# Define the X and Y axis labels
plt.xlabel('Grade Range', fontsize=22, weight='bold', labelpad=5)
plt.ylabel('Occurences', fontsize=22, weight='bold', labelpad=10)

plt.hist(hist_array, bins=bin_ranges, range=(0,100), rwidth=0.75, align='left')
plt.xticks(np.arange(0,91,10),labels=[r'$<$10', r'10s', r'20s',r'30s',r'40s',r'50s',r'60s',r'70s',r'80s', r'$\geq$90']) # label the left edge of each bin
plt.yticks(np.arange(0,17,2))

# uncomment below and set limits if needed
plt.xlim(-5,95)
plt.ylim(0,16)

# Adjust the page layout filling the page using the new tight_layout command
plt.tight_layout(pad=0.5)

# save the figure as a high-res pdf in the current folder
# plt.savefig('plot_filename.pdf')

plt.show()


import os
import shutil
import tarfile
import zipfile

import requests
from tqdm import tqdm

from sally.logger import LogLevel, log
import inquirer

def cleanup_path(dir_path):
    log('clean-up `%s`' % dir_path, LogLevel.INFO)
    if os.path.isdir(dir_path):
        shutil.rmtree(dir_path)
    os.makedirs(dir_path, exist_ok=True)
    log('clean-up is completed `%s`' % dir_path, LogLevel.SUCCESS)

def download_link(path, link):
    return download_request(path, link)

def download_request(path, link):
    response = requests.get(link, stream=True)
    total_length = response.headers.get('content-length')
    log('start to download (size: %s bytes)' % total_length, LogLevel.INFO)

    questions = [
        inquirer.Confirm('install', message='continue to install dataset on your disk?', default = True)
    ]
    answers = inquirer.prompt(questions)
    if not answers['install']:
        log('install process is canceled', LogLevel.ERROR)
        return -1

    with tqdm(total=100) as pbar:
        with open(path, 'wb') as f:
            if total_length is None:
                f.write(response.content)
            else:
                total_length = int(total_length)
                for data in response.iter_content(chunk_size=4096):
                    f.write(data)
                    pbar.update(len(data) / total_length * 100)

    log('download is completed', LogLevel.SUCCESS)

def unzip(path):
    log('extract download file `%s`' % path, LogLevel.INFO)
    if path.endswith('tar.gz') or path.endswith('tar'):
        tar = tarfile.open(path, 'r:gz' if path.endswith('tar.gz') else 'r:')
        tar.extractall(os.path.dirname(path))
        tar.close()
    else:
        zip_ref = zipfile.ZipFile(path)
        zip_ref.extractall(os.path.dirname(path))
        zip_ref.close()

    os.remove(path)
    log('extract is completed', LogLevel.SUCCESS)