import os
import sys
import time
import argparse

import numpy as np

import matplotlib
matplotlib.use('Agg')

from coffea.util import load

from multiprocessing import Pool

from pocket_coffea.utils.configurator import Configurator
from pocket_coffea.utils.plot_utils import PlotManager
from pocket_coffea.parameters.plotting import style_cfg

def slice_accumulator(accumulator, entrystart, entrystop):
    '''Returns an accumulator containing only a reduced set of histograms, i.e. those between the positions `entrystart` and `entrystop`.'''
    _accumulator = dict([(key, value) for key, value in accumulator.items()])
    _accumulator['variables'] = dict(
        [(key, value) for key, value in accumulator['variables'].items()][
            entrystart:entrystop
        ]
    )
    return _accumulator

parser = argparse.ArgumentParser(description='Plot histograms from coffea file')
parser.add_argument('--cfg', default=os.getcwd() + "/config/test.json", help='Config file with parameters specific to the current run', required=False)
parser.add_argument("-i", "--inputfile", required=True, type=str, help="Input file")
parser.add_argument('--plot_dir', default=None, help='Sub-directory inside the plots folder to save plots', required=False)
parser.add_argument('-v', '--version', type=str, default=None, help='Version of output (e.g. `v01`, `v02`, etc.)')
parser.add_argument('-j', '--workers', type=int, default=8, help='Number of parallel workers to use for plotting')
parser.add_argument('-o', '--only', type=str, default='', help='Filter histograms name with string', required=False)
parser.add_argument('-oc', '--only_cat', type=str, default=[''], nargs="+", help='Filter categories with string', required=False)
parser.add_argument('-os', '--only_syst', type=str, nargs="+", default='', help='Filter systematics with a list of strings', required=False)
parser.add_argument('-e', '--exclude', type=str, default=None, help='Exclude categories with string', required=False)
parser.add_argument('--split_systematics', action='store_true', help='Split systematic uncertainties in the ratio plot')
parser.add_argument('--partial_unc_band', action='store_true', help='Plot only the partial uncertainty band corresponding to the systematics specified as the argument `only_syst`')
parser.add_argument('--overwrite', action='store_true', help='Overwrite plots in output folder')
parser.add_argument('--log', action='store_true', help='Set y-axis scale to log')
parser.add_argument('--density', action='store_true', help='Set density parameter to have a normalized plot')
parser.add_argument('-d', '--data_key', type=str, default='DATA', help='Prefix for data samples', required=False)

args = parser.parse_args()
config = Configurator(args.cfg, plot=True, plot_version=args.version)

print("Starting ", end='')
print(time.ctime())
start = time.time()

if os.path.isfile( args.inputfile ): accumulator = load(args.inputfile)
else: sys.exit(f"Input file '{args.inputfile}' does not exist")

if args.plot_dir:
    plot_dir_parent = os.path.dirname(config.plots)
    config.plots = os.path.join(plot_dir_parent, args.plot_dir)

if not args.overwrite:
    if os.path.exists(config.plots):
        raise Exception(f"The output folder '{config.plots}' already exists. Please choose another output folder or run with the option `--overwrite`.")

if not os.path.exists(config.plots):
    os.makedirs(config.plots)

def make_plots(entrystart, entrystop):
    '''Function that instantiates multiple PlotManager objects each managing a different subset of histograms.'''
    _accumulator = slice_accumulator(accumulator, entrystart, entrystop)
    plotter = PlotManager(
        hist_cfg=_accumulator['variables'],
        plot_dir=config.plots,
        style_cfg=style_cfg,
        only_cat=args.only_cat,
        data_key=args.data_key,
        log=args.log,
        density=args.density,
        save=True
    )
    plotter.plot_datamc_all(syst=True, spliteras=False)

# Filter dictionary of histograms with `args.only`
accumulator['variables'] = { k : v for k,v in accumulator['variables'].items() if args.only in k }
if args.exclude:
    accumulator['variables'] = { k : v for k,v in accumulator['variables'].items() if not args.exclude in k }
HistsToPlot = list(accumulator['variables'].keys())

NtotHists = len(HistsToPlot)
NHistsToPlot = len([key for key in HistsToPlot if args.only in key])
print("# tot histograms = ", NtotHists)
print("# histograms to plot = ", NHistsToPlot)
print("Histograms to plot:", HistsToPlot)

# Parallelization of plotting
delimiters = np.linspace(0, NtotHists, args.workers + 1).astype(int)
chunks = [(delimiters[i], delimiters[i+1]) for i in range(len(delimiters[:-1]))]
pool = Pool()
# Parallel calls of make_plots on different subsets of histograms
pool.starmap(make_plots, chunks)
pool.close()

end = time.time()
runTime = round(end-start)
print("Finishing ", end='')
print(time.ctime())
print(f"Drawn {NHistsToPlot} plots in {runTime} s")


"""Setup file for m3u_to_pyradio"""

from setuptools import setup, find_packages

from m3u_to_pyradio_playlist import (
    __project__,
    version,
    __license__,
    __author__,
)

from pathlib import Path

this_directory = Path(__file__).parent
long_description = (this_directory / "README.md").read_text()

meta = dict(
    name=__project__,
    version=version,
    license=__license__,
    description="Converts M3U playlists to pyradio format",
    long_description=long_description,
    long_description_content_type="text/markdown",
    platforms=("Any"),
    author=__author__,
    author_email="rahul.juliato@gmail.com",
    url="https://github.com/LionyxML/pyradio-m3u-to-playlist",
    packages=find_packages(),
    include_package_data=True,
    entry_points={
        "console_scripts": [
            "m3u_to_pyradio=m3u_to_pyradio_playlist.__main__:main"
        ]
    },
    install_requires=["requests"],
    test_suite="tests",
)

if __name__ == "__main__":
    setup(**meta)


# Best run via dedicated term

import time
from multiprocessing import shared_memory, Process, freeze_support, set_start_method
import os

import psutil
import numpy as np
from humanize import naturalsize

#
# Make a big array
# E.g., analagously, this could be a large ML model we want to serve

array_n = 10**4
big_matrix = np.random.rand(array_n, array_n).astype(np.float)

print("array size: ", naturalsize(big_matrix.nbytes))

#
# Create dedicated shared memory object
shared_block = shared_memory.SharedMemory(create=True, size=big_matrix.nbytes)
shared_matrix = np.ndarray(big_matrix.shape, dtype=np.float, buffer=shared_block.buf)
shared_matrix[:] = big_matrix[:]

shared_block_id = shared_block.name

#
# Worker func. Run on forked process
def worker_func(worker_number, matrix_block_id, matrix_shape):
    block = shared_memory.SharedMemory(name=shared_block_id)
    matrix = np.ndarray(matrix_shape, dtype=np.float, buffer=block.buf)
    for _ in range(100):
        print(f"I'm worker {worker_number} with a {matrix.shape[0]}x{matrix.shape[1]} matrix summing to {matrix.sum()}")
        time.sleep(10)

#
# Spawn some processes
if __name__ == "__main__":
    freeze_support()
    set_start_method("fork")

    parent_pid = os.getpid()

    processes = []
    worker_number = 0
    while True:
        print(f"about to fork a proces. will create worker {worker_number}")
        input()

        proc = Process(target=worker_func, args=[worker_number, shared_block_id, big_matrix.shape])
        processes.append(proc)
        proc.start()

        worker_number += 1


import tensorflow as tf
import gym
import gym_sokoban
import numpy as np
import random
import os

state_size = 49
action_size = 8

def softmax(vec):
    return np.exp(x) / np.sum(np.exp(x), axis=0)

def env_loss(y, y_hat):
    alpha = 1
    beta = 300
    reward = y[:,-1]
    reward_hat = y_hat[:,-1]
    reward_loss = tf.losses.mean_squared_error(reward, reward_hat)

    class_true = y[:,:-1]
    class_hat = y_hat[:,:-1]
    class_mat = tf.reshape(class_true, (-1, 49, 7))
    class_mat_hat = tf.reshape(class_hat, (-1, 49, 7))
    # y_hat_mat = [softmax(row) for row in y_hat_mat]
    cross_entropy_loss = tf.nn.softmax_cross_entropy_with_logits_v2(logits=class_mat_hat, labels=class_mat)
    # cross_entropy_loss = tf.reduce_sum(cross_entropy_loss)
    cross_entropy_loss = tf.reduce_mean(cross_entropy_loss)
    combined_loss = tf.add(tf.scalar_mul(alpha, cross_entropy_loss), tf.scalar_mul(beta, reward_loss))
    return combined_loss

def one_hot(arr, dim):
    out = []
    for elem in arr:
        temp = [0]*dim
        temp[elem] = 1
        out.append(temp)
    return np.array(out)

class EnvNN:
    def __init__(self, state_size, action_size):
        def sub_init():
            config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)
            self.sess = tf.Session(config=config)
            self.input_size = state_size + action_size
            self.output_size = state_size*7 + 1
            self.x = tf.placeholder("float32", [None, self.input_size])
            W1 = tf.Variable(tf.random_uniform([self.input_size, 6000], 0, 1))
            b1 = tf.Variable(tf.random_uniform([6000], 0, 1))
            l1 = tf.nn.elu(tf.matmul(self.x, W1)+b1)

            W2 = tf.Variable(tf.random_uniform([6000, 6000], 0, 1))
            b2 = tf.Variable(tf.random_uniform([6000], 0, 1))
            l2 = tf.nn.elu(tf.matmul(l1, W2)+b2)

            W3 = tf.Variable(tf.random_uniform([6000, self.output_size], 0, 1))
            b3 = tf.Variable(tf.random_uniform([self.output_size], 0, 1))
            self.y_hat = tf.nn.elu(tf.matmul(l2, W3)+b3)

            self.y = tf.placeholder("float32", [None, self.output_size])
            # loss = tf.losses.mean_squared_error(self.y, self.y_hat)
            self.loss = env_loss(self.y, self.y_hat)
            self.train = tf.train.AdamOptimizer(0.001).minimize(self.loss)
            self.saver = tf.train.Saver(max_to_keep = 5, keep_checkpoint_every_n_hours = 1)
            self.sess.run(tf.global_variables_initializer())
            self.temp = W1

        self.use_cpu = False
        if self.use_cpu:
            with tf.device('/cpu:0'):
                sub_init()
        else:
            with tf.device('/device:GPU:1'):
                sub_init()

    def update(self, prev_state, action, next_state, reward):
        # print(np.shape(prev_state))
        # print(prev_state)
        x = np.reshape(np.hstack((prev_state,action)), (-1, self.input_size))
        y = np.reshape(np.hstack((next_state,reward)), (-1, self.output_size))
        # print("x: ", x)
        # print("y: ", y)
        self.sess.run(self.train, {self.x: x, self.y: y})

    def predict(self, prev_state, action):
        x = prev_state.append(action)
        reward_vec = self.sess.run(self.y_hat, {self.x: x})
        state = reward_vec[:len(reward_vec) - 1]
        reward = reward_vec[len(reward_vec):]
        return state, reward

class Environment:
    def __init__(self, state_size, action_size):
        self.state_size = state_size
        self.action_size = action_size
        self.episodes = int(1e100)
        self.max_time = 500
        self.model = self._build_model()
        self.init = tf.global_variables_initializer()
        self.checkpoint_dir = './ENVcheckpoints/'

    def _build_model(self):
        tf.reset_default_graph()
        return EnvNN(self.state_size, self.action_size)

    def restore_session(self):
        global_step = 0
        if not os.path.exists(self.checkpoint_dir):
            raise IOError(self.checkpoint_dir + ' does not exist.')
        else:
            path = tf.train.get_checkpoint_state(self.checkpoint_dir)
            if path is None:
                raise IOError('No checkpoint to restore in ' + self.checkpoint_dir)
            else:
                self.model.saver.restore(self.model.sess, path.model_checkpoint_path)
                global_step = int(path.model_checkpoint_path.split('-')[-1])

        return global_step

    def verify(self):
        while True:
            try:
                env = gym.make('TinyWorld-Sokoban-small-v0')
            except RuntimeWarning:
                print("RuntimeWarning caught: retrying")
                continue
            except RuntimeError:
                print("RuntimeError caught: retrying")
                continue
            else:
                break

        while True:
            try:
                state = env.reset()
                state = compress(state)
                prev_state = state

            except RuntimeWarning:
                print("RuntimeWarning caught: retrying")
                continue
            except RuntimeError:
                print("RuntimeError caught: retrying")
                continue
            else:
                break

        done = False
        t = 0
        while not done and t < self.max_time:
            action = random.randint(0,7)
            action_vec = one_hot([action], 8)
            next_state, reward, done, _ = env.step(action)
            print("reward: ", reward)
            print("next state: ", compress(next_state))
            next_state = compress(next_state)
            next_state_vec = np.reshape(one_hot(next_state, 7), (-1,))
            #self.model.update(state, action_vec, np.reshape(one_hot(next_state, 7), (-1,)), reward)
            # print(self.sess.run())
            x = np.reshape(np.append(prev_state,action_vec), (1, self.state_size + self.action_size))
            y = np.reshape(np.append(next_state_vec,reward), (1, self.state_size*7 + 1))
            predicted_state, loss = self.model.sess.run([self.model.y_hat, self.model.loss], {self.model.x: x, self.model.y: y})
            predicted_state = predicted_state[0]
            print("predicted reward: ", predicted_state[-1])
            predicted_state = predicted_state[:-1]
            predicted_state = np.reshape(predicted_state, (-1, 7))
            predicted_state = np.argmax(predicted_state, axis = 1)
            print("predicted_state: ", predicted_state)
            print("predicted loss: ", loss)
            prev_state = state
            state = next_state
            t += 1

    # Should be def train(self, agent_action)
    def train(self, restore_session=False):
        if restore_session:
            self.restore_session()

        while True:
            try:
                env = gym.make('TinyWorld-Sokoban-small-v0')
            except RuntimeWarning:
                print("RuntimeWarning caught: retrying")
                continue
            except RuntimeError:
                print("RuntimeError caught: retrying")
                continue
            else:
                break

        for e in range(self.episodes):
            if e % 10 == 0:
                self.model.saver.save(self.model.sess, self.checkpoint_dir+'model', global_step=e)
                print('Model {} saved'.format(e))

            while True:
                try:
                    state = env.reset()
                    state = compress(state)
                    prev_state = state

                except RuntimeWarning:
                    print("RuntimeWarning caught: retrying")
                    continue
                except RuntimeError:
                    print("RuntimeError caught: retrying")
                    continue
                else:
                    break

            batch = []
            done = False
            t = 0
            while not done and t < self.max_time:
                action = random.randint(0,7)
                action_vec = one_hot([action], 8)
                next_state, reward, done, _ = env.step(action)
                next_state = compress(next_state)
                next_state_vec = np.reshape(one_hot(next_state, 7), (-1,))
                # reward = np.array(reward)

                # print("state ", state)
                # print("action vec ", action_vec)
                # print("next state ", next_state)
                # print("reward ", reward)

                batch.append(np.array([state, action_vec, next_state_vec, reward]))
                #self.model.update(state, action_vec, next_state_vec, reward)
                # print(self.sess.run())
                prev_state = state.copy()
                state = next_state.copy()
                t += 1

            if len(batch) >= 30:
                batch = np.array(batch)
                np.random.shuffle(batch)
                minibatch= np.array(batch[:30])

                mb_state = np.array(minibatch[:,0])
                mb_state = np.array([np.array(lst) for lst in mb_state])

                mb_action = np.array(minibatch[:,1])
                mb_action = np.array([np.array(lst[0]) for lst in mb_action])

                mb_next_state = np.array(minibatch[:,2])
                mb_next_state = np.array([np.array(lst) for lst in mb_next_state])

                mb_reward = np.array(minibatch[:,3])
                mb_reward = np.array([np.array([item]) for item in mb_reward])

                # print("update state ", mb_state)
                # print("update action vec ", mb_action)
                # print("update next state ", mb_next_state)
                # print("update reward ", mb_reward)

                # print("update state shape ", mb_state.shape)
                # print("update action vec shape ", mb_action.shape)
                # print("update next state shape ", mb_next_state.shape)
                # print("update reward shape ", mb_reward.shape)

                self.model.update(mb_state, mb_action, mb_next_state, mb_reward)

                x = np.reshape(np.hstack((mb_state,mb_action)), (-1, self.state_size + self.action_size))
                y = np.reshape(np.hstack((mb_next_state,mb_reward)), (-1, self.state_size*7 + 1))
                print(self.model.sess.run(self.model.loss, {self.model.x: x, self.model.y: y}))
                # print("episode: {}/{}, score: {}".format(e, self.episodes, reward))
                print("episode: {}, score: {}".format(e, reward))
            else:
                pass

def compress(state):
    new_state = []
    for block in state:
        temp = []
        for arr in block:
            if arr[0] == 0:
                temp.append(0)
            elif arr[0] == 243:
                temp.append(1)
            elif arr[0] == 254:
                if arr[1] == 126:
                    temp.append(2)
                if arr[1] == 95:
                    temp.append(3)
            elif arr[0] == 142:
                temp.append(4)
            elif arr[0] == 160:
                temp.append(5)
            elif arr[0] == 219:
                temp.append(6)
        new_state.append(temp)
    new_state = np.array(new_state)
    new_state = new_state.flatten()
    return new_state


import mesa

class Walker(mesa.Agent):
    def __init__(self, unique_id, model, pos, heading=(1, 0)):
        super().__init__(unique_id, model)
        self.pos = pos
        self.heading = heading
        self.headings = {(1, 0), (0, 1), (-1, 0), (0, -1)}

class ShapeExample(mesa.Model):
    def __init__(self, N=2, width=20, height=10):
        self.N = N  # num of agents
        self.headings = ((1, 0), (0, 1), (-1, 0), (0, -1))  # tuples are fast
        self.grid = mesa.space.SingleGrid(width, height, torus=False)
        self.schedule = mesa.time.RandomActivation(self)
        self.make_walker_agents()
        self.running = True

    def make_walker_agents(self):
        unique_id = 0
        while True:
            if unique_id == self.N:
                break
            x = self.random.randrange(self.grid.width)
            y = self.random.randrange(self.grid.height)
            pos = (x, y)
            heading = self.random.choice(self.headings)
            # heading = (1, 0)
            if self.grid.is_cell_empty(pos):
                print(f"Creating agent {unique_id} at ({x}, {y})")
                a = Walker(unique_id, self, pos, heading)
                self.schedule.add(a)
                self.grid.place_agent(a, pos)
                unique_id += 1

    def step(self):
        self.schedule.step()


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import os
from scipy.stats import kurtosis , skew

# save np.load
np_load_old = np.load

# modify the default parameters of np.load
np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k)

X = np.load("X_lstm.npy")
Y = np.load("Y_lstm.npy")

# Finding minimum length LightCurve
min = 10000000000
for e in X:
  if  (min > len(e)):
    min = len(e)

print(min)

#Making every LightCurve same length
xt = []
for e in X:
  xt.append(e[:12901])
X = np.array(xt)
print(X.shape,Y.shape)

mean = np.mean(X,axis=1)
std = np.std(X, axis = 1)
var = np.var(X,axis =1)
kurtosis = kurtosis(X,axis = 1)
skew = skew(X, axis=1)

df = pd.DataFrame()
print(df)
df['lc'] = X.tolist()
df['mean'] = mean
df['std'] = std
df['var'] = var
df['kurtosis'] = kurtosis
df['skew'] = skew
df['class'] = Y
print(df)

df.to_csv('ndata.csv') #Saving ndata.csv


#!/usr/bin/python3
import sys, os
homedir = os.getenv("HOME")
sys.path.append(homedir + '/Pigrow/scripts/')
import pigrow_defs

class sensor_config():
    # find connected sensors
    def find_settings():
        print("connection_type=dt_pin:sck_pin")
        print("connection_address_list=")
        print("default_connection_address=")
        print("available_info=cal_values")
        print("available_settings=cal_zero,cal_known")

    def run_request(request_name, sensor_location, sensor_name=""):
        request_name = request_name.lower()
        if request_name == "cal_values":
            sensor_config.read_cal(sensor_name)
        else:
            print(" Request not recognised")

    def run_setting(setting_string, location, sensor_name=""):
        if "=" in setting_string:
            equals_pos = setting_string.find("=")
            setting_name = setting_string[:equals_pos]
            setting_value = setting_string[equals_pos + 1:]
            if setting_name == "cal_zero":
                sensor_config.cal_zero(location, sensor_name, setting_value)
            if setting_name == "cal_known":
                sensor_config.cal_known(location, sensor_name, setting_value)
        # settings without value

    # Change Settings
    def cal_zero(location, sensor_name, value):
        if sensor_name == "":
            print(" No sensor name supplied, can not calibrate")
            return("No sensor name supplied, can not calibrate")
        # calibrating from sensor or supplied value
        if value == "":
            reading0 = read_sensor(location, raw_only=True)
            if not reading0 == "none":
                offset = float(reading0)
            else:
                return "Sensor failed to read"
        else:
            try:
                offset = float(value)
            except:
                print(" Unable to understand supplied offset value")
                return(" Unable to understand supplied offset value")
        # writing to config file
        text_out = " Zero offset value set to " + str(offset)
        sensor_config.set_extra("zero_offset", offset, sensor_name)
        print(text_out)
        return text_out

    def cal_known(location, sensor_name, value):
        text, extra_settings = sensor_config.read_cal(sensor_name, quiet=True)
        zero_offset = extra_settings['zero_offset']
        if zero_offset == "not set" or zero_offset == "":
            print(" Zero offset must be calibrated first")
            return("Zero offset must be calibrated first")
        if sensor_name == "":
            print(" No sensor name supplied, can not calibrate")
            return("No sensor name supplied, can not calibrate")
        # calibrating from sensor or supplied value
        if not value == "":
            reading0 = read_sensor(location, raw_only=True)
            if not reading0 == "none":
                known_value = float(reading0) - float(zero_offset)
            else:
                return "Sensor failed to read"
        else:
            print(" No value for known grams supplied")
            return(" No value for known grams supplied")
        # writing to config file
        text_out = " calibration value for " + value + " grams set to " + str(known_value)
        sensor_config.set_extra("known_g_value", known_value, sensor_name)
        sensor_config.set_extra("known_grams", value, sensor_name)
        print(text_out)
        return text_out

    # Information Requests - these do not change any settings.
    def read_cal(sensor_name, quiet=False):
        loc_settings = homedir + "/Pigrow/config/pigrow_config.txt"
        extra = pigrow_defs.read_setting(loc_settings, "sensor_" + sensor_name + "_extra")
        if extra == "":
            msg = " - No extra string set for sensor " + sensor_name
            print(msg)
            return msg, {}
        zero_offset =   "Not set"
        known_grams =   "Not set"
        known_g_value = "Not set"
        if ":" in extra:
            settings = extra.split(":")
        else:
            settings = [extra]
        #
        for set in settings:
            if "=" in set:
                key = set.split("=")[0]
                val = set.split("=")[1]
                if key == "zero_offset":
                    zero_offset = val
                if key == "known_grams":
                    known_grams = val
                if key == "known_g_value":
                    known_g_value = val

        text_info = "zero_offset = " + str(zero_offset)
        text_info += "\nKnown value " + str(known_grams) + " grams"
        text_info += "\n            " + str(known_g_value) + " raw sensor value"
        if not quiet == True:
            print(text_info)
        return text_info, {"zero_offset":zero_offset, "known_grams":known_grams, "known_g_value":known_g_value}

    # Read and Write extra string to config
    def set_extra(key, value, sensor_name):
        text, extra_settings = sensor_config.read_cal(sensor_name, quiet=True)
        extra_settings[key]=value
        # make extra string
        extra_string = ""
        for setting, set_value in extra_settings.items():
            extra_string += setting + "=" + str(set_value) + ":"
        extra_string = extra_string[:-1]
        # write to config file
        loc_settings = homedir + "/Pigrow/config/pigrow_config.txt"
        pigrow_defs.change_setting(loc_settings, "sensor_" + sensor_name + "_extra", extra_string)

def read_sensor(location="", extra="", sensor_name="", raw_only=False, *args):
    # read settings
    if not raw_only == True and not sensor_name == "":
        text, extra_settings = sensor_config.read_cal(sensor_name, quiet=True)
        if "zero_offset" in extra_settings:
            try:
                zero_offset = float(extra_settings["zero_offset"])    # raw value when there is no added weight on the sensor
            except:
                print(" Cant convert zero_offset to float, defaulting to zero")
                zero_offset = 0
        if "known_grams" in extra_settings:
            try:
                known_grams = float(extra_settings["known_grams"])    # weight of known value measurement in grams
            except:
                print(" Cant convert known_grams to float, defaulting to zero")
                known_grams = 0
        if "known_g_value" in extra_settings:
            try:
                known_g_value = float(extra_settings["known_g_value"])    # raw value when the known value weight is applied to the sensor
            except:
                print(" Cant convert known_grams to float, defaulting to zero")
                known_g_value = 0

    # find weight from value
    def find_weight(zero_offset, known_grams, known_g_value, value):
        zeroed = value - zero_offset
        raw_per_gram = known_g_value / known_grams
        weight = zeroed / raw_per_gram
        #print("Weight : ", weight, " grams")
        return round(weight, 2)

    # sanity check on values
    def check_measures(measures):
        #print("Testing Measures")
        range_buff = 100
        range_min = measures[0] - range_buff
        range_max = measures[0] + range_buff
        limited_list = []
        for x in measures:
            if x < range_max and x > range_min:
                limited_list.append(x)
        if not len(limited_list) > 1:
            limited_list = []
            range_min = measures[1] - range_buff
            range_max = measures[1] + range_buff
            for x in measures:
                if x < range_max and x > range_min:
                    limited_list.append(x)
            if len(limited_list) == 1:
                #print(" Readings are bad" )
                return "None"
        # got list minus crazy values
        if not len(limited_list) > 3:
            return "None"
        total = 0
        for m in limited_list:
            total = total + m
        average = total / len(limited_list)
        return average
        #print("Average ", average)
        #for m in limited_list:
        #    print(m)

    # Try importing the modules then give-up and report to user if it fails
    import datetime
    import time
    import RPi.GPIO as GPIO
    try:
        from hx711 import HX711
    except:
        print("HX711 module not installed, install using the command;")
        print("   sudo pip3 install HX711 ")
        return None

    # set up and read the sensor
    read_attempt = 1
    if not ":" in location:
        print(" - Location should be in the format 'dt_pin:sck_pin' ")
        return None
    dt_pin = int(location.split(":")[0])
    sck_pin = int(location.split(":")[1])
    hx711 = HX711(dout_pin=dt_pin, pd_sck_pin=sck_pin, channel='A', gain=64)
    measures = "None"
    while read_attempt < 5:
        try:
            hx711.reset()
            measures = hx711.get_raw_data()
            valid_result = check_measures(measures)
            #
            if valid_result == "None":
                print("--problem reading HX711, try " + str(read_attempt))
                time.sleep(2)
                read_attempt = read_attempt + 1
            else:
                if raw_only == True:
                    return valid_result
                logtime = datetime.datetime.now()
                if not known_grams == 0 and not known_g_value == 0:
                    weight = find_weight(zero_offset, known_grams, known_g_value, valid_result)
                else:
                    weight = "uncalibrated"
                GPIO.cleanup()
                data = [['time',logtime], ['weight', weight], ['average',valid_result]]
                for pos in range(0, len(measures)):
                    data.append(["reading" + str(pos), measures[pos]])
                return data
        except Exception as e:
            print("--exception while reading HX711, try " + str(read_attempt))
            print(" -- " + str(e))
            time.sleep(2)
            read_attempt = read_attempt + 1
    GPIO.cleanup()
    return None

if __name__ == '__main__':
    '''
      The HX711 has two wires that need gpio address
      '''
     # check for command line arguments
    sensor_location = ""
    sensor_name = ""
    request = ""
    setting_string = ""
    for argu in sys.argv[1:]:
        if "=" in argu:
            equals_pos = argu.find("=")
            thearg = argu[:equals_pos]
            thevalue = argu[equals_pos + 1:]
            if thearg == 'location':
                sensor_location = thevalue
            if thearg == 'request':
                request = thevalue
            if thearg == 'set':
                setting_string = thevalue
            if thearg == "name":
                sensor_name = thevalue
        elif 'help' in argu or argu == '-h':
            print(" Modular control for HX711 weight sensor interface")
            print(" ")
            print(" Location should be in the format 'dt pin:sck_pin")
            print("")
            print(" -config  ")
            print("        display the config information")
            print("")
            sys.exit(0)
        elif argu == "-flags":
            print("location=dt_pin:sck_pin")
            sys.exit(0)
        elif argu == "-config":
            sensor_config.find_settings()
            sys.exit()

    if not request == "":
        sensor_config.run_request(request, sensor_location, sensor_name)
        sys.exit()
    if not setting_string == "":
        sensor_config.run_setting(setting_string, sensor_location, sensor_name)
        sys.exit()

    # read sensor
    #if not sensor_location == "":
    output = read_sensor(location=sensor_location, sensor_name=sensor_name)
    #else:
    #    print(" No sensor address supplied, this requries a sensor address.")
    #    sys.exit()
    #
    if output == None:
        print("!! Failed to read !!")
    else:
        for x in output:
            print(str(x[0]) + "=" + str(x[1]))


from sklearn.metrics import classification_report
from data_processing.data_splitting import get_leave_one_out_splits
import tensorflow as tf
import numpy as np

# Define a custom loss function that ignores the padding in the labels (unlabeled pixels)
def masked_sparse_categorical_crossentropy(y_true, y_pred):
    y_true = tf.cast(y_true, tf.int32)
    mask = tf.not_equal(y_true, 0)
    y_true_adjusted = y_true - 1
    y_true_adjusted = tf.where(mask, y_true_adjusted, 0)

    loss = tf.keras.losses.sparse_categorical_crossentropy(y_true_adjusted, y_pred, from_logits=False, axis=-1)
    mask = tf.cast(mask, loss.dtype)
    loss *= mask

    return tf.reduce_mean(loss)

# Define a custom accuracy metric that ignores the padding in the labels (unlabeled pixels)
def masked_sparse_categorical_accuracy(y_true, y_pred):
    y_true = tf.cast(y_true, tf.int32)
    mask = tf.not_equal(y_true, 0)
    y_true_adjusted = y_true - 1
    y_true_adjusted = tf.where(mask, y_true_adjusted, 0)

    matches = tf.cast(tf.equal(y_true_adjusted, tf.argmax(y_pred, axis=-1, output_type=tf.int32)), tf.float32)
    mask = tf.cast(mask, matches.dtype)
    matches *= mask
    return tf.reduce_sum(matches) / tf.reduce_sum(mask)

# Train and evaluate the model using leave-one-out cross-validation
def train_and_evaluate(model_builder, num_classes, X, y, epochs, batch_size, reshape=False):

    scores = []
    y_true = []
    y_pred = []

    num_samples = X.shape[0]
    input_shape = (X.shape[1], X.shape[2], X.shape[3]) if len(X.shape) > 3 else (X.shape[1], X.shape[2])

    train_indices, test_indices = get_leave_one_out_splits(X,y)

    for train_index, test_index in zip(train_indices, test_indices):

        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]

        model = model_builder(input_shape=input_shape, num_classes=num_classes)

        model.compile(optimizer='adam', loss= masked_sparse_categorical_crossentropy, metrics=[masked_sparse_categorical_accuracy])

        model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1)
        score = model.evaluate(X_test, y_test, verbose=0)
        scores.append(score)

        # Store true labels and predictions for classification report
        y_true.extend(np.ravel(y_test))
        y_pred.extend(np.argmax(model.predict(X_test), axis=-1).ravel())

    # Calculate average performance metrics
    avg_score = np.mean(scores, axis=0)

    # Calculate classification report only for labelled pixels
    filtered_y_true = [value for value in y_true if value != 0]
    filtered_y_pred = [pred for pred, true in zip(y_pred, y_true) if true != 0]
    report = classification_report(filtered_y_true, filtered_y_pred)

    return avg_score, report, y_true, y_pred


#!/usr/bin/env python
import datetime
from functools import partial
from glob import glob
import json
import os
import re
import shlex
import shutil
import sys

import torch
from torch.utils.data import DataLoader
from tqdm import tqdm

from igms.datasets import get_dataset
from igms.featurize import load_featurizer
from igms.generator import make_generator
from igms.mmd import mmd2, Estimator
from igms.kernels import pick_kernel
from igms.utils import ArgumentParser, get_optimizer, pil, set_other_seeds, str2bool

def parse_args(argv=None, check_dir=True):
    parser = ArgumentParser()
    parser.add_argument("outdir")
    parser.add_argument(
        "--force", action="store_true", help="Delete OUTDIR if it exists."
    )

    parser.add_argument(
        "--resume",
        action="store_true",
        help=(
            "Resume the last checkpoint from OUTDIR. "
            "Overrides all other arguments except DEVICE and DATA_WORKERS."
        ),
    )

    ds = parser.add_argument_group("dataset options")
    ds.add_argument("--dataset", default="celebA")
    ds.add_argument("--out-size", type=int, default=64)
    ds.add_argument("--out-channels", type=int, default=3)
    ds.add_argument("--download-dataset", type=str2bool, default=True)

    gen = parser.add_argument_group("generator options")
    gen.add_argument("--z-dim", type=int, default=128)
    gen.add_argument("--batch-size", type=int, default=128)
    gen.add_argument("--generator", default="dcgan")

    opt = parser.add_argument_group("optimizer options")
    opt.add_argument("--optimizer", default="adam:1e-4")
    opt.add_argument("--epochs", type=int, default=25)

    loss = parser.add_argument_group("loss options")
    loss.add_argument("--featurizer", default="through:smoothing")
    loss.add_argument("--image-noise", type=float, default=0)
    loss.add_argument("--kernel", default="linear")
    loss.add_argument(
        "--mmd-estimator",
        default="unbiased",
        type=lambda s: s.upper().replace("-", "_"),
    )

    comp = parser.add_argument_group("computation options")
    comp.add_argument("--data-workers", type=int, default=2)
    comp.add_argument("--device", default="cuda")
    args = parser.parse_args(argv)

    if args.resume:
        checkpoints = glob(f"{args.outdir}/checkpoint_*.pt")
        if not checkpoints:
            parser.error(f"Couldn't find any checkpoints to resume in {args.outdir}")

        pat = re.compile(r"/checkpoint_(\d+)_(\d+)\.pt$")

        def checkpoint_idx(s):
            return tuple(int(x) for x in pat.search(s).groups())

        checkpoint = torch.load(
            max(checkpoints, key=checkpoint_idx), map_location=args.device
        )
        checkpoint_args = checkpoint["args"]
        checkpoint_args.device = args.device
        checkpoint_args.data_workers = args.data_workers
        checkpoint_args.resume = True
        return checkpoint_args, checkpoint
    elif check_dir:
        if os.path.exists(args.outdir):
            ls = set(os.listdir(args.outdir))
            if args.force or ls.issubset({"args.json", "argv.txt"}):
                shutil.rmtree(args.outdir)
            else:
                parser.error(
                    f"{args.outdir} already exists; "
                    "use --resume, or --force to delete"
                )
        os.makedirs(args.outdir)

        v = parser._read_args_from_files(sys.argv[1:] if argv is None else argv)
        with open(os.path.join(args.outdir, "argv.txt"), "x") as f:
            print(
                "# This file has the effective arguments passed, one per line.\n"
                "# Note there might be duplicates (later taking precedence).\n"
                "# You can run them again with "
                f"`{os.path.basename(__file__)} @path/to/argv.txt`.\n"
                "# See args.json for how they were parsed by argparse.\n",
                file=f,
            )
            for arg in v:
                print(shlex.quote(arg), file=f)
        with open(os.path.join(args.outdir, "args.json"), "w") as f:
            json.dump(vars(args), f, indent=2)

    return args, {}

def main():
    args, checkpoint = parse_args()

    device = torch.device(args.device)
    if device.type == "cuda":
        assert torch.cuda.is_available()
        assert torch.backends.cudnn.enabled
        torch.backends.cudnn.benchmark = True

    generator = make_generator(
        args.generator,
        output_size=args.out_size,
        z_dim=args.z_dim,
        output_channels=args.out_channels,
    ).to(device)
    opt = get_optimizer(args.optimizer)(generator.parameters())

    if checkpoint:
        generator.load_state_dict(checkpoint["generator_state_dict"])
        opt.load_state_dict(checkpoint["optimizer_state_dict"])
        log_latents = checkpoint["log_latents"].to(device)
        resume = (checkpoint["epoch"], checkpoint["batches"])
    else:
        log_latents = torch.empty(64, generator.z_dim, device=device)
        log_latents.uniform_(-1, 1)
        resume = None

    featurizer = load_featurizer(args.featurizer, map_location=device).to(device)
    top_kernel = pick_kernel(args.kernel)
    estimator = Estimator[args.mmd_estimator]

    if args.image_noise:
        real_noise = torch.empty(
            args.batch_size, *generator.output_shape, device=device
        )
        fake_noise = torch.empty(
            args.batch_size, *generator.output_shape, device=device
        )

    def loss_fn(real_imgs, fake_imgs):
        if args.image_noise:
            real_imgs = real_imgs + real_noise.normal_(std=args.image_noise)
            fake_imgs = fake_imgs + fake_noise.normal_(std=args.image_noise)
        real_reps = featurizer(real_imgs)
        fake_reps = featurizer(fake_imgs)
        return mmd2(top_kernel(real_reps, fake_reps), estimator=estimator)

    dataset = get_dataset(
        args.dataset, out_size=args.out_size, download=args.download_dataset
    )
    dataloader = DataLoader(
        dataset,
        batch_size=args.batch_size,
        pin_memory=True,
        shuffle=True,
        drop_last=True,
        num_workers=args.data_workers,
        worker_init_fn=set_other_seeds,
    )

    train_loop(args, dataloader, generator, loss_fn, opt, log_latents, resume)

def train_loop(args, dataloader, generator, loss_fn, opt, log_latents, resume=None):
    fn = partial(os.path.join, args.outdir)
    device = args.device

    latents = torch.empty(args.batch_size, generator.z_dim, device=device)
    generator.train()

    with open(fn("log.txt"), "a", buffering=1) as log_f:

        def write_line(line):
            s = f"[{datetime.datetime.now():%Y-%m-%d %H:%m:%S}]  {line}"
            tqdm.write(s)
            log_f.write(s + "\n")

        def checkpoint(epoch, batches=0):
            d = {
                "args": args,
                "generator_state_dict": generator.state_dict(),
                "optimizer_state_dict": opt.state_dict(),
                "log_latents": log_latents,
                "epoch": epoch,
                "batches": batches,
            }

            name = orig_name = f"checkpoint_{epoch}_{batches}.pt"
            while os.path.exists(fn(name)):
                name += "~"
            torch.save(d, fn(name))

            if name != orig_name:
                write_line(
                    f"ERROR: checkpoint {orig_name} already exists! "
                    f"Saved to {name} instead, now aborting."
                )
                sys.exit(3)
            else:
                write_line(f"saved checkpoint: {name}")

        def save_samples(name):
            generator.eval()
            file = f"fake_{name}.jpg"
            pil(generator(log_latents)).save(fn(file))
            generator.train()
            write_line(f"saved samples: {file}")

        if resume:
            resume_epoch, resume_batch = resume
            epochs = range(resume_epoch, args.epochs)
            skip_batches = resume_batch + 1
            write_line(f"Resuming from {resume_epoch} / {resume_batch}")
        else:
            pil(next(iter(dataloader))[0][: log_latents.shape[0]]).save(fn("real.jpg"))
            checkpoint(0)
            epochs = range(args.epochs)
            skip_batches = 0

        for epoch in tqdm(epochs, unit="epoch", dynamic_ncols=True):
            with tqdm(dataloader, unit="batch", dynamic_ncols=True) as bar:
                for batch_i, (real_imgs, real_ys) in enumerate(bar):
                    if skip_batches:
                        skip_batches -= 1
                        continue

                    real_imgs = real_imgs.to(device)

                    opt.zero_grad()
                    latents.uniform_(-1, 1)
                    fake_imgs = generator(latents)

                    loss = loss_fn(real_imgs, fake_imgs)
                    loss.backward()
                    opt.step()

                    l_str = f"{loss.item():,.3f}"
                    bar.set_postfix(loss="=" + l_str.rjust(11))

                    if batch_i % 100 == 0:
                        write_line(f"{epoch:>3} / {batch_i:>7,}: {l_str:>14}")

                    if batch_i % 500 == 0:
                        save_samples(f"{epoch}_{batch_i}")
            checkpoint(epoch, batch_i)

        write_line(f"{epoch:>3} / {batch_i:>7,}: {l_str:>14}")
        save_samples(f"{epoch}_{batch_i}")

if __name__ == "__main__":
    main()


import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import numpy as np

def get_ckd_dataset(normalized=True,test_ratio=0.2):
    ckd_df = pd.read_csv('ckd.csv')
    numerical_columns = set(['age','bp','bgr','bu','sc','sod','pot','hemo','pcv','wbcc','rbcc'])
    nominal_columns = set(['sg','al','su','rbc','pc','pcc','ba','htn','dm','cad','appet','pe','ane','class'])
    all_columns = set(ckd_df.columns)
    numeric_ckd_names = list(all_columns.intersection(numerical_columns))
    nominal_ckd_names = list(all_columns.intersection(nominal_columns))

    for col_name in numeric_ckd_names:
        col = ckd_df[col_name]
        col.fillna(np.mean(col),inplace=True)

    for col_name in nominal_ckd_names:
        col = ckd_df[col_name]
        col = col.apply(str)

    # to replace ? in each column with the random values following the value distribution of that column
    for nominal_name in nominal_ckd_names:
        norminal_col =  ckd_df[nominal_name]
        counts = norminal_col.value_counts()
        counts_keys = counts.keys()
        valid_value_list = [k for k in counts_keys if k != '?']
        valid_value_count = np.array([counts[k] for k in counts_keys if k != '?'])
        valid_value_prob = valid_value_count * 1.0 / np.sum(valid_value_count)
        num_of_unknow_records = len(norminal_col[norminal_col == '?'])
        norminal_col[norminal_col == '?'] = np.random.choice(valid_value_list,num_of_unknow_records,replace=True,p=valid_value_prob)

    nominal_data = ckd_df[nominal_ckd_names].drop('class',axis=1)
    true_categories = ckd_df['class']
    true_categories.replace('ckd',1,inplace=True)
    true_categories.replace('notckd',0,inplace=True)
    nominal_data_encoded = pd.get_dummies(nominal_data,drop_first=True)
    numeric_data = ckd_df[numeric_ckd_names]

    X_ckd = pd.concat([numeric_data,nominal_data_encoded],axis=1)
    X_ckd = X_ckd.astype(float)
    y_ckd = true_categories.values
    y_ckd = y_ckd.reshape((1,len(y_ckd)))
    y_ckd = y_ckd.astype(float)

    if not normalized:
      X_train, X_test, y_train, y_test = train_test_split(X_ckd, y_ckd.T, test_size=test_ratio, random_state=100)
      return X_train, X_test, y_train, y_test

    scaler = StandardScaler()
    X_ckd_normalized = scaler.fit_transform(X_ckd)
    X_train, X_test, y_train, y_test = train_test_split(X_ckd_normalized, y_ckd.T, test_size=test_ratio, random_state=100)

    return X_train, X_test, y_train, y_test


import unittest
from unittest import skip
from typing import cast
from pathlib import Path

import pandas as pd

from synthetic_table_metrics.data_container import Data
from synthetic_table_metrics.synthetic_table_metrics import SyntheticTableMetrics

class TestIrisTableMetrics(unittest.TestCase):
    @staticmethod
    def test_on_copy():
        real = pd.read_csv(Path("data", "iris_original.csv"))
        synthetic = pd.read_csv(Path("data", "iris_original.csv"))
        data = Data(real, synthetic)
        metrics = SyntheticTableMetrics()
        result = metrics.run(data)
        ks = result.keys()
        assert "detectability" in ks

class TestSyntheticTableMetrics(unittest.TestCase):
    @staticmethod
    def test_on_census_dataset():
        # The census dataset is larger then the dataset used to develop
        # this package. The size of the dataset caused an error, this
        # is why we explicitely test on this dataset.
        real = pd.read_csv(Path("data", "census.csv"))
        synthetic = cast(
            pd.DataFrame,
            pd.read_csv(Path("data", "census.csv")),
        )
        # Take a subset of the data
        synthetic = synthetic.head(200)

        data = Data(real, synthetic)

        metrics = SyntheticTableMetrics()
        metrics = metrics.run(data)

        # Check if result contains the correct keys
        ks = list(metrics.keys())
        assert "detectability" in ks
        assert "duplicates" in ks


from pathlib import Path
import numpy as np
import matplotlib.image as mpimg
import scipy.signal as scs

# scales for both thresholds [0,255]
# threshold for defining same-colored rows (RGB_rowmax - RGB_rowmin < threshold for each of the three RGB components)
color_threshold = 50
# threshold for edge detection on 0: no edge (no gradient of color), 255: max.possible gradient
edge_threshold = 100

# length of filter for vertical edge detection
filter_length = 2 # -1 -1 1 1

# remove all rows composed of pixels of the same color
def remove_equal_rows(im):

    [h,w,d] = im.shape
    rm = np.ones(h)

    for i in range(h):
        crowmin = np.ndarray.min(im[i,:,:], axis=0)
        crowmax = np.ndarray.max(im[i,:,:], axis=0)

        if (np.all((crowmax - crowmin) < 50)):
            rm[i] = 0

    #print(np.sum(rm.astype(int)))
    imr = np.zeros((np.sum(rm.astype(int)), w, d), 'uint8')
    cnt = 0
    imr = im[rm > 1e-2,:,:]

    return imr

# on some pics the images of before/after are surrounded by frames -> remove those
def remove_image_contour(im):
    # check horizontal rows
    im = remove_equal_rows(im)
    # and vertical columns
    im = np.transpose(im, (1,0,2))
    im = remove_equal_rows(im)
    return np.transpose(im, (1,0,2))

# assumption 1 x n symmetric filter
def apply_filter_rowwise(im, filter):
    [h,w,d] = im.shape
    im_edge = np.zeros((h,w))
    # move a filter over the image
    for i in range(h):
        for k in range(d):
            im_edge[i,:] += np.power(np.convolve(im[i,:,k], filter, 'same'), 2)

    im_edge = np.power(im_edge, 0.5)
    # rescale the results for each column to [0,255]
    for j in range(w):
        im_edge[:,j] = 255 * (im_edge[:,j] / np.max(im_edge)) # / np.max(im_edge[:,j]))
    return im_edge

def func(x):
    t = 20

    rv = np.logical_and(x[:,0] > 20, x[:,1] > 20)
    rv = np.logical_and(rv, x[:,2] > 20)
    return rv

def vertical_split(im):
    [h,w,d] = im.shape

    #filter = np.zeros(filter_length)
    #filter[0] = -1
    #filter[1] = 1
    #filter[2] = -1
    #filter[3] = 1
    #filter[4] = 1
    #filter[5] = 1
    #im_edge = apply_filter_rowwise(im, filter)

    #filter = np.hstack( (-np.ones((50,1)), np.zeros((50,1)), np.ones((50,1))) )

    filter = np.zeros((3,3))
    filter[0,0] = -1; filter[0,2] = 1
    filter[1,0] = -2; filter[1,2] = 2
    filter[2,0] = -1; filter[2,2] = 1

    im = im.astype(float)
    im_temp = np.zeros_like(im)
    for c in range(d):
        im_temp[:,:,c] = np.absolute(scs.convolve2d(im[:,:,c], filter, mode='same'))

    im_temp /= np.max(im_temp)
    '''
    print(np.shape( np.linalg.norm(im_temp, axis=2) ))
    mpimg.imsave("data_prepared/" + "___check__sum__" + filename, im_temp[:,:,0])#np.linalg.norm(im_temp,axis=2))

    im_sum = np.zeros((1,w))
    for j in range(10,w-10,1):
        im_sum[0,j] = np.sum( func( np.absolute(im[:,j,:]-im[:,j-1,:]) ) ) / h

    #print(im_sum[0,0:10])
    '''
    im_temp_norm = np.linalg.norm(im_temp,axis=2)
    mpimg.imsave("data_prepared/" + "___check__sum__" + filename, im_temp_norm>0.05) #np.vstack( im_temp_norm/np.max(im_temp_norm) > 120, im_sum )) )

    im_sum = np.zeros(w)
    for j in range(10,w-10,1):
        for c in range(d):
            im_sum[j] += np.sum(np.absolute(im[:,j,c] - im[:,j-1,c]))

        #im_sum[j] = np.sum((im_temp_norm[:,j] > 0.05))/h

    idx = np.argmax(im_sum)
    val = im_sum[idx]

    im1 = im[:,0:idx,:]
    im2 = im[:,idx+1:-1,:]

    im_sum[idx-10:idx+10:1] = -val
    val2 = np.max(im_sum)

    if (2*val2 > val):
        mpimg.imsave("data_cut/" + "cut_" + filename, np.hstack((im1.astype('uint8'), im2.astype('uint8'))))
    else:
        mpimg.imsave("data_cut/" + "cut_" + filename, np.hstack((im1.astype('uint8'), np.zeros((h,40,d)).astype('uint8'), im2.astype('uint8'))))
    im_sum = np.sort(im_sum)
    print(im_temp_norm.shape)
    print(im_sum.shape)
    print(im_sum[range(w-10,w,1)])
    ims = []
    '''
    prj = 0
    for j in range(filter_length, w-filter_length, 1):
        if np.all(im_edge[:,j] > edge_threshold):
            ims.append(im[:,prj:j,:])
            j += 2*filter_length # to avoid very narrow "artefact" images, in case the border is not perfect
    '''
    return ims

# split the picture in any number of parts by ONLY vertical or horizaontal cuts, and return all of them
def split_picture(im):
    ims = vertical_split(im)
    # no vertical split found -> try horizontal
    if (len(ims) == 1):
        im = np.transpose(im, (1,0,2))
        ims = vertical_split(im)
        for i in range(len(ims)):
            ims[i] = np.transpose(ims[i], (1,0,2))

    if (len(ims) == 2):
        print("Successful split!")
    return ims

if __name__ == "__main__":

    # open the file containing initial dataset and read its header
    input = open('VisualBMI_cvs_dump_small.txt', 'r')
    input.readline()

    # iterate throgh all samples and check if each one is valid
    cnt_iter = 0
    for line in input:
        # check if we donwloaded the file
        props = line.split()
        filename = props[6] + ".jpg"
        img_file = Path("data/" + filename)
        if img_file.is_file():
            # load, remove all countours and overwrite
            print(filename)
            img = mpimg.imread("data/" + filename)
            img = remove_image_contour(img)
            split_picture(img)
            mpimg.imsave("data_prepared/" + filename, img)
        cnt_iter += 1
        #print(cnt_iter, " samples processed.")
    input.close()
'''
    props = line.split()
    d = {}
    d["id"] = 1
    d["weight"] = int(props[1])
    d["height"] = int(props[3])

    dd = {}
    dd["id"] = 2
    dd["weight"] = int(props[2])
    dd["height"] = int(props[3])

    dall = [d,dd]

    json.dump(dall,output)
    output.close()

    output = open('VisualBMI_extracted.txt', 'r')
    data = json.load(output)
'''


import math
import random
import wavio
import numpy as np
import scipy.fftpack
import scipy.io as sio
import scipy.io.wavfile
from playsound import playsound as ps
import struct
import display_plot
import wave as wav
import convert_sixteen
import os
from shutil import copyfile
import genetic as gen
import display_plot as disp

N = 1000

def get_clips(dir_name, max_clips):
  clips = [f for f in os.listdir(dir_name) if '.wav' in f]
  fft_data = []
  if max_clips == 1:
    clips = [clips[np.random.randint(0, len(clips))]]
    # clips = [clip for clip in clips if clip == 'Curly_Wurlie.3.wav']
    print(f'Goal clip = {clips[0]}')
    copyfile(dir_name+clips[0], os.getcwd()+'/mod_clips/exp0_goal_'+clips[0])
  for clip in range(min(max_clips, len(clips))):
    wv = dir_name + clips[clip]
    _, data = sio.wavfile.read(wv)
    # transformed_data = np.array(scipy.fftpack.rfft(data))
    # fft_data.append(transformed_data)
    fft_data.append(data)
  return np.array(fft_data)

counter = 0

def play_fft(fft_vector):
    global counter
    counter += 1
    cwd = os.getcwd()
    file_path = cwd+'/mod_clips/exp' + str(counter) + '.wav'
    write_file(file_path, fft_vector)
    return

def write_file(name, sample_data):
  sio.wavfile.write(filename=name, rate=44100, data=sample_data)
  convert_sixteen.convert_single_file(name)
  os.remove(name)

def get_temp(n):

  T = N/n
  rand = np.random.uniform(low=0, high=0.69)
  if (math.exp(rand/T)-1) > 1:
    print(f'annealing problem! anneal val: {(math.exp(rand/T)-1)}')
    return 1
  return math.exp(rand/T)-1

def breed_loop(N, data, gdata, save_interval):
  ffts = []
  min_errs = []
  for n in range(N):
    # print("Generation:", n)
    errors, goal = gen.fit_all(data, gdata)
    new_data = np.zeros(np.shape(data))
    for i in range(len(data)//2):
      if goal:
        print("Goal!")

      r = np.arange(len(errors))
      i1, i2 = np.random.choice(r, 2, p=errors)
      cross1, cross2 = gen.crossover(data[i1], data[i2])

      cross1, cross2 = gen.mutate(cross1, get_temp(n+1)), gen.mutate(cross2, get_temp(n+1))
      # cross1, cross2 = gen.mutate(cross1, .5), gen.mutate(cross2, .5)
      new_data[i], new_data[i+(len(data)//2)] = cross1, cross2

      if n % save_interval == 0 and i == 0:
        play_fft(cross1)

    data = new_data
    if n % 25 == 0:
        print(f'min error {np.amin(errors)}')
        min_errs.append(np.amin(errors))

  return ffts, min_errs

def main():
  print('deleting files in mod_clips')
  mods = os.getcwd()+'/mod_clips/'
  for f in os.listdir(mods):
    os.remove(mods+f)
  dir_name = os.getcwd() + '/1secs/'
  g_dir_name = os.getcwd() + '/1sec_goals/'
  print('getting ffts from clips')
  data = get_clips(dir_name, 8657)
  np.random.shuffle(data)
  gdata = get_clips(g_dir_name, 1)
  ffts, min_errs = breed_loop(N, data, gdata, save_interval=20)
  # disp.display_ffts(ffts, len(ffts[0]))
  disp.display_fitness(min_errs, len(min_errs))

main()


from scipy.io.wavfile import read
import matplotlib.pyplot as plt

from utils.Helpers import load_audio

path_wav = "../../DATA/train_dir/train/wav/id10001/1zcIwhmdeo4/00001.wav"

input_data = read(path_wav)
audio = input_data[1]
# plot the first 1024 samples
plt.plot(audio[0:208])
# label the axes
plt.ylabel("Amplitude")
plt.xlabel("Time")
# set the title
plt.title("Raw Waveform")
# display the plot
plt.show()

import numpy
import scipy.io.wavfile

sample_rate, signal = scipy.io.wavfile.read(path_wav)  # File assumed to be in the same directory
signal = signal[0:int(2 * sample_rate)]
pre_emphasis = 0
emphasized_signal = numpy.append(signal[0], signal[1:] - pre_emphasis * signal[:-1])

plt.plot(signal)
plt.subplots_adjust(left=0.15,
                    bottom=0.1,
                    right=0.9,
                    top=0.9,
                    wspace=0.5,
                    hspace=0.5)

plt.title("Vergleich Pre-Emphasis")
plt.xlabel('Anzahl Samples')
plt.ylabel('Frequenz')
plt.plot(emphasized_signal)
plt.legend(['ohne Pre-Emphasis', 'Pre-Emphasis = 0.97'])
plt.show()


"""
Use XGBosst to predict clinical only data (tabular)
"""

from xgboost import XGBClassifier
import pandas as pd
from pathlib import Path
import pickle
from sklearn.metrics import roc_auc_score
import optuna, os
import numpy as np
import matplotlib.pyplot as plt
from pytorch_tabular.models import TabNetModel, TabNetModelConfig
from pytorch_tabular.models.tab_transformer import TabTransformerConfig
from pytorch_tabular.models.ft_transformer import FTTransformerConfig
from pytorch_tabular.models.autoint import AutoIntConfig
from pytorch_tabular.models import CategoryEmbeddingModelConfig

from pytorch_tabular import TabularModel
from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig, ExperimentConfig

# MODEL = "tabtransformer"
# MODEL = "FTtransformer"
MODEL = "autoint"

data_dir = Path("/home/jiashu/seq/raw_data")
processed_data_dir = Path("/home/jiashu/seq/processed/train30_test10")

df = pd.read_excel(
    data_dir / "Dissection Gesture_Clinical Data.xlsx",
    engine='openpyxl'
)

df['Nerve Sparing'] = df['Nerve Sparing'].apply(
    lambda val: 1 if val == 'full' else 0
)
X = df.drop(["Case ID", "Surgeon ID", "Pre-Op SHIM",], axis=1)

data_config = DataConfig(
    target=['ESI @ 12 mo'],
    continuous_cols=["Age", "BMI", "CCI", "PSA", "Prostate volume (g)", "Pre-op Gleason", "Post-op Gleason"],
    categorical_cols=[ "ASA", "Nerve Sparing", "ECE", 'Radiation after surgery 1=Yes, 0=No', 'Postop ADT'],
)

trainer_config = TrainerConfig(
    auto_lr_find=False, # Runs the LRFinder to automatically derive a learning rate
    batch_size=128,
    deterministic=True,
    # disable ckpt
    checkpoints=None,
    max_epochs=15,
    gpus=1, #index of the GPU to use. 0, means CPU
)
optimizer_config = OptimizerConfig()

# model_config = TabNetModelConfig(
#     task="classification",
#     metrics=["accuracy"],
#     learning_rate=1e-3,
# )
# model_config = CategoryEmbeddingModelConfig(
#     task="classification",
#     layers="1024-512-512",  # Number of nodes in each layer
#     activation="LeakyReLU", # Activation between each layers
#     learning_rate = 1e-3,
#     metrics=["accuracy"],
#     metrics_params=[{}]
# )

def train_model(model_config, k_fold) -> float:
    tabular_model = TabularModel(
        data_config=data_config,
        model_config=model_config,
        optimizer_config=optimizer_config,
        trainer_config=trainer_config,
    )

    k_dir = processed_data_dir / f"k{k_fold}"
    with open(k_dir / "train.pkl", "rb") as f:
        seq = pickle.load(f)
    train_idxs = df['Case ID'].isin(seq.keys()).values
    X_train = X[train_idxs]

    with open(k_dir / "valid.pkl", "rb") as f:
        seq = pickle.load(f)
    test_idxs = df['Case ID'].isin(seq.keys()).values
    X_test = X[test_idxs]

    tabular_model.fit(train=X_train, validation=X_test)
    pred_df = tabular_model.predict(X_test)
    prob = pred_df['1_probability']
    try:
        return roc_auc_score(X_test['ESI @ 12 mo'], prob)
    except:
        return 0.0

def get_auc_over_k_folds(model_config):
    rets = []
    for k_fold in range(4):
        auc = train_model(model_config, k_fold)
        rets.append(auc)
    return np.mean(rets)

def objective_tabtransformer(trial):
    emb_dim = trial.suggest_categorical("input_embed_dim", [16, 32, 64, 128, 256])
    emb_dropout = trial.suggest_float("embedding_dropout", 0.1, 0.6)
    learning_rate = trial.suggest_float("lr", 1e-5, 1e-1, log=True)
    share_emb = trial.suggest_categorical("share_embedding", [True, False])
    ff = trial.suggest_categorical("out_ff_layers", ["128-64-32", "64-32-16", "128-128-32"])
    bn_norm = trial.suggest_categorical("use_batch_norm", [True, False])

    args = dict(
        input_embed_dim=emb_dim,
        embedding_dropout=emb_dropout,
        learning_rate=learning_rate,
        share_embedding=share_emb,
        out_ff_layers=ff,
    )
    if bn_norm:
        args = {
            **args,
            "use_batch_norm": True,
            "batch_norm_continuous_input": True
        }

    model_config = TabTransformerConfig(
        task="classification",
        metrics=["accuracy"],
        seed=42,
        **args
    )

    return get_auc_over_k_folds(model_config)

def objective_fttransformer(trial):
    args = dict(
        learning_rate=trial.suggest_float("learning_rate", 1e-5, 1e-1, log=True),
        input_embed_dim=trial.suggest_categorical("input_embed_dim", [16, 32, 64, 128, 256]),
        embedding_dropout=trial.suggest_float("embedding_dropout", 0.1, 0.6),
        share_embedding=trial.suggest_categorical("share_embedding", [True, False]),
        num_attn_blocks=trial.suggest_int("num_attn_blocks", 2, 10),
        attn_dropout=trial.suggest_float("attn_dropout", 0.1, 0.6),
    )
    model_config = FTTransformerConfig(
        task="classification",
        metrics=["accuracy"],
        seed=42,
        **args
    )

    return get_auc_over_k_folds(model_config)

def objective_autoint(trial):
    args = dict(
        learning_rate=trial.suggest_float("learning_rate", 1e-5, 1e-1, log=True),
        attn_embed_dim=trial.suggest_categorical("attn_embed_dim", [8, 16, 32]),
        num_attn_blocks=trial.suggest_int("num_attn_blocks", 2, 5),
        attn_dropouts=trial.suggest_float("attn_dropouts", 0.1, 0.6),
        has_residuals=trial.suggest_categorical("has_residuals", [True, False]),
        embedding_dim=trial.suggest_categorical("embedding_dim", [16, 32, 64]),
        embedding_dropout=trial.suggest_float("embedding_dropout", 0.1, 0.6),
        deep_layers=trial.suggest_categorical("has_residuals", [True, False]),
    )
    model_config = AutoIntConfig(
        task="classification",
        metrics=["accuracy"],
        seed=42,
        **args
    )

    return get_auc_over_k_folds(model_config)

models_objectives = {
    "tabtransformer": objective_tabtransformer,
    "FTtransformer": objective_fttransformer,
    "autoint": objective_autoint
}

print(f"model = {MODEL}, using objective {models_objectives[MODEL]}")

study = optuna.create_study(direction='maximize')
study.optimize(models_objectives[MODEL], n_trials=3000)

trial = study.best_trial

print(f"best study has avg acc over k-fold = {trial.value}")
for key, value in trial.params.items():
  print("    {}: {}".format(key, value))


from collections import OrderedDict

import numpy as np
import pydantic
import pytest
from conftest import dir_save_location_path

from releso.shape_parameterization import ShapeDefinition, VariableLocation
from releso.util.logger import VerbosityLevel

@pytest.mark.parametrize(
    "current_position, error",
    [
        (0, False),
        (100, False),
        (-100, False),
        (0.001, False),
        (-0.001, False),
        (None, True),  # Need at least current_position to instantiate
    ],
)
def test_variable_location_instantiates_only_current_position(
    current_position, error, dir_save_location
):
    """Test that VariableLocation instantiates"""
    # First check the error state.
    cal_dict = {
        "current_position": current_position,
        "save_location": dir_save_location,
    }
    if error:
        with pytest.raises(pydantic.ValidationError):
            variable_location = VariableLocation(**cal_dict)
        return
    # check default initiation
    variable_location = VariableLocation(**cal_dict)
    assert variable_location is not None
    assert variable_location.current_position == current_position
    assert variable_location.is_action() is False
    assert variable_location.min_value == current_position
    assert variable_location.max_value == current_position
    assert variable_location.n_steps == 10
    assert current_position == variable_location._original_position

@pytest.mark.parametrize(
    "current_position, min_value, max_value, error",
    [
        (0, None, None, False),
        (0, -10, None, False),
        (-10, -12, -8, False),
        (-10, -8, None, True),
        (-10, -8, -7, True),
        (-10, None, -12, True),
    ],
)
def test_variable_location_min_max_location(
    current_position, min_value, max_value, error, dir_save_location
):
    """Test for correct setting of the min and max values."""
    # check if error is emitted
    cal_dict = {
        "current_position": current_position,
        "min_value": min_value,
        "max_value": max_value,
        "save_location": dir_save_location,
    }
    if error:
        with pytest.raises(pydantic.ValidationError) as exc_info:
            variable_location = VariableLocation(**cal_dict)
        assert "or equal to the" in str(exc_info.value)
        return
    variable_location = VariableLocation(**cal_dict)
    if min_value is None:
        assert current_position == variable_location.min_value
    else:
        assert min_value == variable_location.min_value
    if max_value is None:
        assert current_position == variable_location.max_value
    else:
        assert max_value == variable_location.max_value

@pytest.mark.parametrize(
    "current_position, min_value, max_value, is_action",
    [
        (0, -1, 1, True),
        (0, 0, 1, True),
        (0, None, None, False),
        (0, -1, 0, True),
    ],
)
def test_variable_location_is_action(
    current_position, min_value, max_value, is_action, dir_save_location
):
    variable_location = VariableLocation(
        **{
            "current_position": current_position,
            "min_value": min_value,
            "max_value": max_value,
            "save_location": dir_save_location,
        }
    )
    assert variable_location.is_action() is is_action

@pytest.mark.parametrize(
    "current_position, min_value, max_value, n_steps, step, expected_step, "
    "error",
    [
        (0, -1, 1, None, None, 0.2, False),
        (0, -1, 1, None, 0.1, 0.1, False),
        (0, -2, 2, 100, None, 4 / 100, False),
        (0, -2, 2, 100, 0.25, 0.25, False),
    ],
)
def test_variable_location_step_length(
    current_position,
    min_value,
    max_value,
    n_steps,
    step,
    expected_step,
    error,
    dir_save_location,
):
    """Test for correct setting of the n_steps and step values."""
    # check if error is emitted
    cal_dict = {
        "current_position": current_position,
        "min_value": min_value,
        "max_value": max_value,
        "n_steps": n_steps,
        "step": step,
        "save_location": dir_save_location,
    }
    if error:
        with pytest.raises(pydantic.ValidationError) as exc_info:
            variable_location = VariableLocation(**cal_dict)
        assert "or equal to the" in str(exc_info.value)
        return
    variable_location = VariableLocation(**cal_dict)
    assert np.isclose(expected_step, variable_location.step)
    if n_steps is None and step is None:
        assert n_steps == variable_location.n_steps

@pytest.mark.parametrize(
    "current_position, min_value, max_value, step, warning",
    [
        (0, -4, 1, 0.2, False),
        (0, -1, 1, 4, True),  # step length is greater than the interval
        (0, -2, 2, 4, False),
        (0, -2, 2, 4.0001, True),  # step length is greater than the interval
    ],
)
def test_variable_location_step_length_warning(
    current_position,
    min_value,
    max_value,
    step,
    warning,
    caplog,
    dir_save_location,
):
    cal_dict = {
        "current_position": current_position,
        "min_value": min_value,
        "max_value": max_value,
        "step": step,
        "save_location": dir_save_location,
    }
    if warning:
        with caplog.at_level(VerbosityLevel.WARNING, logger="ReLeSO_parser"):
            VariableLocation(**cal_dict)
            assert "is greater than the interval for" in caplog.text
    else:
        VariableLocation(**cal_dict)

@pytest.mark.parametrize(
    "current_position, min_value, max_value, n_steps, step, iterations, "
    "expect_position",
    [
        (
            0,
            None,
            None,
            None,
            None,
            [True] * 5,
            0,
        ),  # stay in bound upper limit
        (
            0,
            None,
            None,
            None,
            None,
            [False] * 5,
            0,
        ),  # stay in bound lower limit
        (0, -10, None, None, None, [True] * 5, 0),  # stay in bound upper limit
        (
            0,
            -10,
            None,
            None,
            None,
            [False] * 5,
            -5,
        ),  # correct value after iteration low
        (
            0,
            None,
            10,
            None,
            None,
            [True] * 5,
            5,
        ),  # correct value after iteration up
        (0, None, 10, None, None, [False] * 5, 0),  # lower bound w boundary
        (0, -10, None, None, None, [True] * 15, 0),  # upper bound w boundary
        (
            0,
            -10,
            None,
            None,
            None,
            [False] * 15,
            -10,
        ),  # lower bound w boundary
        (0, None, 10, None, None, [True] * 15, 10),  # upper bound w boundary
        (0, None, 10, None, None, [False] * 15, 0),  # lower bound w boundary
        (0, -1, 4, None, None, [True] * 3, 1.5),  # complex case
        (
            0,
            -1,
            4,
            None,
            None,
            [False, False, False, True, True, True, True, True],
            1.5,
        ),  # complex case back and forth
        (
            0,
            -1,
            4,
            None,
            None,
            [
                True,
                True,
                True,
                True,
                True,
                True,
                True,
                True,
                True,
                True,
                True,
                False,
            ],
            3.5,
        ),  # complex case back and forth
    ],
)
def test_variable_location_discrete_action(
    current_position,
    min_value,
    max_value,
    n_steps,
    step,
    iterations,
    expect_position,
    dir_save_location,
):
    variable_location = VariableLocation(
        **{
            "current_position": current_position,
            "min_value": min_value,
            "max_value": max_value,
            "n_steps": n_steps,
            "step": step,
            "save_location": dir_save_location,
        }
    )
    last_current_position = current_position
    for iteration in iterations:
        last_current_position = variable_location.apply_discrete_action(
            iteration
        )
    assert np.isclose(
        last_current_position, variable_location.current_position
    )
    assert np.isclose(expect_position, variable_location.current_position)
    # check if original position is correctly preserved
    assert current_position == variable_location._original_position
    # check if reset works as expected
    variable_location.reset()
    assert variable_location._original_position == current_position
    assert variable_location.current_position == current_position

@pytest.mark.parametrize(
    "current_position, min_value, max_value, n_steps, step, iterations, "
    "expect_position",
    [
        # iterations are values in range [-1,1] defining the value_range of
        # the variable_location
        (10, 5, 15, None, None, [1], 15),  # upper limit
        (10, 5, 15, None, None, [0], 10),  # middle, start from middle
        (10, 5, 15, None, None, [-1], 5),  # lower limit
        (0, None, None, None, None, [1.1, -1.1], 0),  # stay in bound
        (0, -10, None, None, None, [0, 2], 0),  # stay in bound upper
        (
            0,
            -10,
            None,
            None,
            None,
            [2, 0],
            -5,
        ),  # two step correct value exceed upper
        (
            0,
            None,
            10,
            None,
            None,
            [-2, 0],
            5,
        ),  # two step correct value exceed lower
        (0, None, 10, None, None, [0.8, -1.4], 0),  # two step lower limit
        (0, -10, None, None, None, [-1.8, 1], 0),  # two step upper limit
        (0, -1, 4, None, None, [-0.33, -0.467999999], 0.33),  # complex value
    ],
)
def test_variable_location_continuous_action(
    current_position,
    min_value,
    max_value,
    n_steps,
    step,
    iterations,
    expect_position,
    dir_save_location,
):
    variable_location = VariableLocation(
        **{
            "current_position": current_position,
            "min_value": min_value,
            "max_value": max_value,
            "n_steps": n_steps,
            "step": step,
            "save_location": dir_save_location,
        }
    )
    last_current_position = current_position
    for iteration in iterations:
        last_current_position = variable_location.apply_continuous_action(
            iteration
        )
    assert np.isclose(
        last_current_position, variable_location.current_position
    )
    assert np.isclose(expect_position, variable_location.current_position)
    # check if original position is correctly preserved
    assert current_position == variable_location._original_position
    # check if reset works as expected
    variable_location.reset()
    assert variable_location._original_position == current_position
    assert variable_location.current_position == current_position

# test ShapeDefinition

@pytest.mark.parametrize(
    "control_points, n_elements, error",
    [
        ([[0.0], [1.0], [1.0]], 3, False),
        (
            [[0.0, 0.1, 0.2, 0.3, 0.4], [1.0], [1.0]],
            15,
            False,
        ),  # this might change
        ([[1.0]], 1, False),
        ([[(1.0, 2.0)]], 1, True),
        (
            [
                [
                    VariableLocation(
                        current_position=1,
                        save_location=dir_save_location_path(),
                    )
                ]
            ],
            1,
            False,
        ),
        (
            [
                [
                    VariableLocation(
                        current_position=1,
                        save_location=dir_save_location_path(),
                    ),
                    1.0,
                ]
            ],
            2,
            False,
        ),
        (
            [
                [
                    {
                        "current_position": 1,
                        "save_location": dir_save_location_path(),
                    }
                ]
            ],
            1,
            False,
        ),
        (
            [
                [
                    OrderedDict(
                        {
                            "current_position": 1,
                            "save_location": dir_save_location_path(),
                        }
                    )
                ]
            ],
            1,
            False,
        ),
    ],
)
def test_shape_definition_initiation(
    control_points, n_elements, error, dir_save_location
):
    # for idx in range(len(control_points)):
    #     for idy in range(len(control_points[idx])):
    #         if isinstance(control_points[idx][idy], dict):
    #             control_points[idx][idy]["save_location"] = dir_save_location
    #             control_points[idx][idy] = VariableLocation(
    #                 **control_points[idx][idy]
    #             )
    cal_dict = {
        "control_points": control_points,
        "save_location": dir_save_location,
    }
    if error:
        with pytest.raises(pydantic.ValidationError) as err:
            shape = ShapeDefinition(**cal_dict)
            assert "either a float or a VariableLocation" in str(err.value)
        return
    shape = ShapeDefinition(**cal_dict)
    all_variables_variable_location = True
    for element in shape.control_points:
        for item in element:
            if not isinstance(item, VariableLocation):
                all_variables_variable_location = False
                break
        if not all_variables_variable_location:
            break
    assert all_variables_variable_location

    assert shape.get_number_of_points() == n_elements

    cps = shape.get_control_points()
    all_the_same = True
    for original_cp_list, cp_list in zip(control_points, cps):
        for point_o, point in zip(original_cp_list, cp_list):
            if not (
                (
                    isinstance(point_o, VariableLocation)
                    and point_o.current_position == point
                )
                or (
                    isinstance(point_o, (dict, OrderedDict))
                    and point_o["current_position"] == point
                )
                or point_o == point
            ):
                all_the_same = False
                break
        if not all_the_same:
            break
    assert all_the_same

    assert shape.get_shape() == shape.get_control_points()

@pytest.mark.parametrize(
    "control_points, n_actions",
    [
        (
            [
                [
                    {
                        "current_position": 0.1,
                        "min_value": 0,
                        "save_location": "test",
                    },
                    {
                        "current_position": 0.2,
                        "min_value": 0,
                        "save_location": "test",
                    },
                    {
                        "current_position": 0.3,
                        "min_value": 0,
                        "save_location": "test",
                    },
                    {
                        "current_position": 0.4,
                        "min_value": 0,
                        "save_location": "test",
                    },
                    {
                        "current_position": 0.5,
                        "min_value": -1,
                        "save_location": "test",
                    },
                ]
            ],
            5,
        ),
        (
            [
                [
                    {"current_position": 0.1, "save_location": "test"},
                    {"current_position": 0.2, "save_location": "test"},
                    {
                        "current_position": 0.3,
                        "min_value": 0,
                        "save_location": "test",
                    },
                    {
                        "current_position": 0.4,
                        "min_value": 0,
                        "save_location": "test",
                    },
                    {
                        "current_position": 0.5,
                        "min_value": -1,
                        "save_location": "test",
                    },
                ]
            ],
            3,
        ),
    ],
)
def test_shape_definition_get_actions(
    control_points, n_actions, dir_save_location
):
    new_control_points = []
    for points in control_points:
        new_cps = []
        for point in points:
            point["save_location"] = dir_save_location
            new_cps.append(VariableLocation(**point))
        new_control_points.append(new_cps)
    shape = ShapeDefinition(
        **{
            "control_points": new_control_points,
            "save_location": dir_save_location,
        }
    )
    actions = shape.get_actions()
    assert len(actions) == n_actions

    original_cps = shape.get_control_points()

    for action in actions:
        action.apply_continuous_action(0.2)

    for o_cp, cp in zip(original_cps, shape.get_control_points()):
        assert o_cp != cp

    shape.reset()

    for o_cp, cp in zip(original_cps, shape.get_control_points()):
        assert o_cp == cp


from django.http import Http404
from django.shortcuts import render, redirect

from django.contrib.auth.forms import UserCreationForm
from rest_framework import status, permissions, generics
from rest_framework.response import Response
from rest_framework.views import APIView

from moodtracker.models import MoodInput,MoodForm

# Create your views here.
from moodtracker.permissions import IsOwnerOrReadOnly
from moodtracker.serializers import MoodInputSerializer, UserSerializer

#Adapted and Copied from: https://www.django-rest-framework.org/tutorial/
class UserList(generics.ListAPIView):
    queryset = MoodInput.objects.all()
    serializer_class = UserSerializer

class MoodList(APIView):
    permission_classes = [permissions.IsAuthenticatedOrReadOnly, IsOwnerOrReadOnly]
    def get(self, request, format=None):
        if request.user.is_authenticated:

            ls = MoodInput.objects.filter(user=request.user).order_by('-date')
            setStreakValues(ls)
            serializer = MoodInputSerializer(ls, many=True)
            return Response(serializer.data)
        else:
            return redirect("/login/")
    def post(self,request, format=None):
        serializer = MoodInputSerializer(data=request.data)
        if serializer.is_valid():
            setStreakValues(serializer)
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED1)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
    def perform_create(self, serializer):
        serializer.save(user=self.request.user)

class MoodInputDetail(APIView):
    def get_object(self, pk):
        try:
            return MoodInput.objects.get(pk=pk)
        except MoodInput.DoesNotExist:
            raise Http404

    def get(self, request, pk, format=None):
        moodinput = self.get_object(pk)
        serializer = MoodInputSerializer(moodinput)
        return Response(serializer.data)

    def put(self, request, pk, format=None):
        moodinput = self.get_object(pk)
        serializer = MoodInputSerializer(moodinput, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, pk, format=None):
        moodinput = self.get_object(pk)
        moodinput.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

#Adapted and copied from
def register(response):
    if response.method == "POST":
        form = UserCreationForm(response.POST)
        if form.is_valid():
            form.save()
            return redirect('/moodUserFriendly/')
    else:
        form = UserCreationForm()
    return render(response, 'registration/register.html', context={"form":form})

def home(response):
    ls = []
    if response.method == "POST":
        form = MoodForm(response.POST)
        if form.is_valid():
            form.save(response.user)
        try:
            ls = MoodInput.objects.filter(user=response.user).order_by('-date')
        except:
            pass
    else:
        try:
            ls = MoodInput.objects.filter(user=response.user).order_by('-date')
        except:
            pass
        form = MoodForm()
    setStreakValues(ls)

    return render(response, 'home.html', context={"form":form, "ls":ls})

def setStreakValues(ls):
    lsList = list(ls)
    i = 0
    j = 0
    k = 1
    while k < len(lsList) and j < len(lsList) and i < len(lsList):
        currentStreakModel = lsList[i]

        date0 = lsList[j].date
        date1 = lsList[k].date

        if (date0 - date1).days <= 1:
            currentStreakModel.streak += 1

            j += 1
            k += 1
        else:
            i = k
            j = i
            k = j + 1

def redirecttomood(response):
    return redirect('/moodUserFriendly/')


import streamlit as st
import matplotlib.pyplot as plt
from nii import Image3D
import time
tick = time.time()
image = Image3D("avg152T1_LR_nifti.nii.gz")
print("Time taken: {}".format(time.time()-tick))
option_ph = st.empty()
filename_ph = st.empty()
slider_ph1 = st.empty()
slider_ph2 = st.empty()
image_ph = st.empty()
def main():
    global image
    path = filename_ph.text_input("File name",'avg152T1_LR_nifti.nii.gz')
    option = option_ph.selectbox(
                                    'Select view mode',
                                    ('Manual', 'Automatic')
                                )
    if image.path != path:
        image.update_path(path)
    if(len(image.nii_data.shape)==3):
        if option == 'Manual':
            print(image.nii_data.shape[2],image.nii_data.shape[1])
            sl1 = slider_ph1.slider('Select y_dim', 0, image.nii_data.shape[2],0,1)
            sl2 = slider_ph2.slider('Select x_dim', 0, image.nii_data.shape[1],0,1)
            fig = image.plot_both_slices(sl2,sl1)
            image_ph.pyplot(fig)
        else:
            for i in range(0,image.nii_data.shape[2]):
                fig = image.plot_from_current_slice()
                sl1 = slider_ph1.slider('Select slide', 0, image.nii_data.shape[2],image.current_silce,1)
                image_ph.pyplot(fig)
                image.increment_slice_number()

if __name__ == "__main__":
    main()


from django.db import models

from redactor.fields import RedactorField

# Create your models here.

class SobreNos(models.Model):
	titulo = models.CharField(u'Titulo', max_length=100)
	descricao = RedactorField(u'Descriçao')

	class Meta:
		verbose_name = 'Sobre nós'
		verbose_name_plural = 'Sobre nós'

	def __str__(self):
		return self.titulo


from pathlib import Path

from app.utils import (
    read_parquet_cached,
    get_forecast_data,
)

TEST_DATA_DIR = Path(__file__).resolve().parent.parent / "test_data"

class TestApp:
    def test_get_points_data(self):
        df = read_parquet_cached(TEST_DATA_DIR / "points.pq")
        assert not df.empty

    def test_get_playing_data(self):
        df = read_parquet_cached(TEST_DATA_DIR / "playing.pq")
        assert not df.empty

    def test_get_forecast_data(self):
        df = get_forecast_data(
            TEST_DATA_DIR / "points.pq",
            TEST_DATA_DIR / "playing.pq",
            TEST_DATA_DIR / "points_features.pq",
        )
        assert not df.empty


import libvirt
from vm_control.vm_control import VMControl

class VMLibvirtControl(VMControl):
    def __init__(self):
        self.conn = None
        self.connect()

    def connect(self):
        self.conn = libvirt.open()
        if self.conn is None:
            raise Exception("Failed to open connection to the hypervisor")

    def start_vm(self, vm_name):
        try:
            dom = self.conn.lookupByName(vm_name)
            if dom is None:
                raise Exception(f"VM {vm_name} does not exist")
            if dom.create() < 0:
                raise Exception(f"Failed to start VM {vm_name}")
            print(f"VM {vm_name} started successfully")
        except libvirt.libvirtError as e:
            raise Exception(f"Error starting VM {vm_name}: {e}")

    def stop_vm(self, vm_name):
        try:
            dom = self.conn.lookupByName(vm_name)
            if dom is None:
                raise Exception(f"VM {vm_name} does not exist")
            if dom.destroy() < 0:
                raise Exception(f"Failed to stop VM {vm_name}")
            print(f"VM {vm_name} stopped successfully")
        except libvirt.libvirtError as e:
            raise Exception(f"Error stopping VM {vm_name}: {e}")

    def restart_vm(self, vm_name):
        self.stop_vm(vm_name)
        self.start_vm(vm_name)

    def create_vm(self, vm_name, **kwargs):
        pass

    def delete_vm(self, vm_name):
        pass

    def get_vm_status(self, vm_name):
        try:
            dom = self.conn.lookupByName(vm_name)
            if dom is None:
                raise Exception(f"VM {vm_name} does not exist")
            info = dom.info()
            return info[0]
        except libvirt.libvirtError as e:
            raise Exception(f"Error getting status of VM {vm_name}: {e}")


#!/usr/bin/env python
#
# test_start_end.py - Tests starting/ending sessions
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#

import time

import wx

from . import run_with_wx, yield_until

from wxnat import XNATBrowserPanel

def test_start_end():
    run_with_wx(_test_start_end)
def _test_start_end():

    parent = wx.GetTopLevelWindows()[0]
    panel  = XNATBrowserPanel(parent)
    result = [None]

    def callback(val):
        result[0] = val

    panel.StartSession('not.a.xnat.server',
                       showError=False,
                       callback=callback)

    yield_until(lambda : result[0] is not None)

    assert not result[0]
    assert not panel.SessionActive()

    result[0] = None

    panel.StartSession('central.xnat.org',
                       showError=False,
                       callback=callback)

    yield_until(lambda : result[0] is not None)

    assert result[0]
    assert panel.SessionActive()
    panel.EndSession()
    assert not panel.SessionActive()


import os

import discord
from discord import app_commands as cmds
from discord.ext import commands as discord_commands

import commands
from settings import *

bot = discord_commands.Bot(
    command_prefix='!',
    intents=discord.Intents.all()
)

@bot.event
async def on_ready():

    print(LOGIN_MESSAGE.format(bot=bot.user))

    await bot.change_presence(
        status=discord.Status.online,
        activity=discord.Activity(
            name=BOT_ACTIVITY,
            type=discord.ActivityType.listening,
        )
    )

    try:
        synced = await bot.tree.sync()
        print(SYNC_COMMANDS_MESSAGE.format(n=len(synced)))
    except Exception as exception:
        print(exception)

@bot.tree.command(name=CREATE_ID_COMMAND, description=CREATE_ID_DESCRIPTION)
@cmds.describe(
    nombre=NAME_DESCRIPTION,
    apellido=SURNAME_DESCRIPTION,
    nacimiento=BIRTH_DESCRIPTION,
    genero=GENDER_DESCRIPTION
)
async def create_id(
    interaction: discord.Interaction,
    nombre: str,
    apellido: str,
    nacimiento: str,
    genero: str
):
    print(COMMAND_LOG.format(command=CREATE_ID_COMMAND, user=interaction.user))
    await commands.create_id(interaction, nombre, apellido, nacimiento, genero)

@bot.tree.command(name=SHOW_ID_COMMAND, description=SHOW_ID_DESCRIPTION)
@cmds.describe(usuario=USER_DESCRIPTION)
async def show_id(interaction: discord.Interaction, usuario: discord.User = None):
    print(COMMAND_LOG.format(command=SHOW_ID_COMMAND, user=interaction.user))
    await commands.show_id(interaction, usuario)

@bot.tree.command(name=EDIT_ID_COMMAND, description=EDIT_ID_DESCRIPTION)
@cmds.describe(
    usuario=USER_DESCRIPTION,
    nombre=NAME_DESCRIPTION,
    apellido=SURNAME_DESCRIPTION,
    nacimiento=BIRTH_DESCRIPTION,
    genero=GENDER_DESCRIPTION
)
async def edit_id(
    interaction: discord.Interaction,
    usuario: discord.User,
    nombre: str = '',
    apellido: str = '',
    nacimiento: str = '',
    genero: str = ''
):
    print(COMMAND_LOG.format(command=EDIT_ID_COMMAND, user=interaction.user))
    await commands.edit_id(interaction, usuario, nombre, apellido, nacimiento, genero)

@bot.tree.command(name=REMOVE_ID_COMMAND, description=REMOVE_ID_DESCRIPTION)
@cmds.describe(
    usuario=USER_DESCRIPTION
)
async def remove_id(
    interaction: discord.Interaction,
    usuario: discord.User
):
    print(COMMAND_LOG.format(command=REMOVE_ID_COMMAND, user=interaction.user))
    await commands.remove_id(interaction, usuario)

@bot.tree.command(name=HELP_ID_COMMAND, description=HELP_ID_DESCRIPTION)
async def help_id(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=HELP_ID_COMMAND, user=interaction.user))
    await commands.help_id(interaction)

@bot.tree.command(name=POLL_COMMAND, description=POLL_DESCRIPTION)
async def poll(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=POLL_COMMAND, user=interaction.user))
    await commands.poll(interaction)

@bot.tree.command(name=SERVER_OPEN_COMMAND, description=SERVER_OPEN_DESCRIPTION)
async def server_open(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=SERVER_OPEN_COMMAND, user=interaction.user))
    await commands.server_open(interaction)

@bot.tree.command(name=SERVER_CLOSE_COMMAND, description=SERVER_CLOSE_DESCRIPTION)
async def server_close(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=SERVER_CLOSE_COMMAND, user=interaction.user))
    await commands.server_close(interaction)

@bot.tree.command(name=RESET_COMMAND, description=RESET_DESCRIPTION)
async def reset(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=RESET_COMMAND, user=interaction.user))
    await commands.reset(interaction)

@bot.tree.command(name=MAINTENANCE_COMMAND, description=MAINTENANCE_DESCRIPTION)
async def maintenance(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=MAINTENANCE_COMMAND, user=interaction.user))
    await commands.maintenance(interaction)

@bot.tree.command(name=GREEN_ALERT_COMMAND, description=GREEN_ALERT_DESCRIPTION)
async def green_alert(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=GREEN_ALERT_COMMAND, user=interaction.user))
    await commands.green_alert(interaction)

@bot.tree.command(name=YELLOW_ALERT_COMMAND, description=YELLOW_ALERT_DESCRIPTION)
async def yellow_alert(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=YELLOW_ALERT_COMMAND, user=interaction.user))
    await commands.yellow_alert(interaction)

@bot.tree.command(name=ORANGE_ALERT_COMMAND, description=ORANGE_ALERT_DESCRIPTION)
async def orange_alert(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=ORANGE_ALERT_COMMAND, user=interaction.user))
    await commands.orange_alert(interaction)

@bot.tree.command(name=RED_ALERT_COMMAND, description=RED_ALERT_DESCRIPTION)
async def red_alert(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=RED_ALERT_COMMAND, user=interaction.user))
    await commands.red_alert(interaction)

@bot.tree.command(name=NEUTRAL_ALERT_COMMAND, description=NEUTRAL_ALERT_DESCRIPTION)
async def neutral_alert(interaction: discord.Interaction):
    print(COMMAND_LOG.format(command=NEUTRAL_ALERT_COMMAND, user=interaction.user))
    await commands.neutral_alert(interaction)

if __name__ == '__main__':
    os.system('cls' if os.name == 'nt' else 'clear')
    if DISCORD_LOGS:
        bot.run(DISCORD_TOKEN)
    else:
        bot.run(DISCORD_TOKEN, log_handler=None)