import time
import numpy as np
import tools
import warnings

from typing import Dict, List, Tuple

_BIG_NUMBER = int(1e9)

State = Dict[str, np.ndarray]
Action = List[List[int]]
Info = Dict[str, str]

class Environment:

    def step(self, solution: Action) -> Tuple[State, int, bool, Info]:
        raise NotImplementedError()

    def reset(self, seed: int = None, instance: State = None, epoch_tlim: int = None, is_static: bool = None) -> State:
        raise NotImplementedError()

class ControllerEnvironment(Environment):
    """
    Environment for VRP with Time Windows that wraps interface with controller through the command line
    """

    def __init__(self, read_pipe, write_pipe):
        super().__init__()
        self.read_pipe = read_pipe
        self.write_pipe = write_pipe

    def step(self, solution: Action) -> Tuple[State, int, bool, Info]:
        result = tools.json_loads_np(self._request('step', solution))

        observation = result['observation']
        reward = result['reward']
        done = result['done']
        info = result['info']

        return (observation, reward, done, info)

    def reset(self, seed: int = None, instance: State = None, epoch_tlim: int = None, is_static: bool = None) -> (State):
        assert seed is None, "Argument seed must be None for controller environment as it is decided by controller"
        assert instance is None, "Argument instance must be None for controller environment as it is decided by controller"
        assert epoch_tlim is None, "Argument epoch_tlim must be None for controller environment as it is decided by controller"
        assert is_static is None, "Argument is_static must be None for controller environment as it is decided by controller"
        result = tools.json_loads_np(self._request('reset', None))
        return result['observation'], result['info']

    def _request(self, action: str, data=None):
        self.write_pipe.write(tools.json_dumps_np({
            'action': action,
            'data': data
        }))
        self.write_pipe.write('\n')
        self.write_pipe.flush()
        return self.read_pipe.readline().strip()

class VRPEnvironment(Environment):
    """
    Dynamic environment for VRP with Time Windows (VRPTW) where orders arrive during the day
    An instance is created by resampling customers, time windows and demands from a static VRPTW instance.
    Requests arrive during a number of epochs. At every epoch, a solution can be submitted that dispatches
    a number of vehicles to serve a subset of the open requests. Some requests must be served during the
    current epoch, because the next epoch will be too late to meet there time windows, taking into account
    dispatching time and driving time. Other requests may be served, and may be included in the routes, but
    may also be deferred to the next epoch. During the final epoch, all request must be dispatched.
    The environment has a wall clock time limit to submit solutions. To avoid timing issues, submissions can
    be 'submitted' to the environment early, after which they can still be updated until the time limit.
    A submitted solution is checked and can be accepted or rejected depending on whether it is valid or not.
    To move to the next epoch, the agent should explicitly call the 'next_epoch' function, after which the
    last accepted solution becomes final, and new requests are sampled for the next epoch.
    """

    def __init__(self, seed: int = 1, instance: State = None, epoch_tlim: int = 120, is_static: bool = False):
        super().__init__()
        # DO NOT CHANGE THESE PARAMETERS
        self.MAX_REQUESTS_PER_EPOCH = 100  # Every epoch, we will sample at most 100 new requests
        self.MARGIN_DISPATCH = 3600  # Assume it takes one hour to dispatch the vehicle
        self.EPOCH_DURATION = 3600  # We will dispatch vehicles once an hour
        self.TLIM_GRACE_PERIOD = 2  # 2 seconds wall clock grace time

        # Fill environment with defaults
        self.default_instance = instance
        self.default_seed = seed
        self.default_epoch_tlim = epoch_tlim
        self.default_is_static = is_static

        # Require reset to be called first by marking environment as done
        self.is_done = True
        self.reset_counter = 0

    def reset(self, seed: int = None, instance: State = None, epoch_tlim: int = None, is_static: bool = None) -> State:
        """Resets the environment. Defaults provided during construction can be overridden to reuse the environment."""
        if self.reset_counter > 0 and seed is None:
            warnings.warn("Repeatedly resetting the environment without providing a seed will use the same default seed again")
        self.reset_counter += 1
        self.instance = instance if instance is not None else self.default_instance
        self.seed = seed if seed is not None else self.default_seed
        self.epoch_tlim = epoch_tlim if epoch_tlim is not None else self.default_epoch_tlim
        self.is_static = is_static if is_static is not None else self.default_is_static

        assert self.instance is not None

        if self.is_static:
            self.start_epoch = 0
            self.current_epoch = 0
            self.end_epoch = 0
        else:
            self.rng = np.random.default_rng(self.seed)
            timewi = self.instance['time_windows']
            self.start_epoch = int(max((timewi[1:, 0].min() - self.MARGIN_DISPATCH) // self.EPOCH_DURATION, 0))
            self.end_epoch = int(max((timewi[1:, 0].max() - self.MARGIN_DISPATCH) // self.EPOCH_DURATION, 0))

            self.current_epoch = self.start_epoch

            # Initialize request array with dummy/padding/sentinel request for depot
            self.request_id = np.array([0])
            self.request_customer_index = np.array([0])
            self.request_timewi = self.instance['time_windows'][0:1]
            self.request_service_t = self.instance['service_times'][0:1]
            self.request_demand = self.instance['demands'][0:1]
            self.request_is_dispatched = np.array([False])
            self.request_epoch = np.array([0])
            self.request_must_dispatch = np.array([False])

        self.is_done = False
        obs = self._next_observation()

        self.final_solutions = {}
        self.final_costs = {}
        self.start_time_epoch = time.time()

        info = {
            # For the dynamic problem, requests will be sampled from the static instance
            'dynamic_context': self.instance if not self.is_static else None,
            'is_static': self.is_static,
            'start_epoch': self.start_epoch,
            'end_epoch': self.end_epoch,
            'num_epochs': self.end_epoch - self.start_epoch + 1,
            'epoch_tlim': self.epoch_tlim,
        }

        return obs, info

    def step(self, solution: Action) -> Tuple[State, int, bool, Info]:

        assert not self.is_done, "Environment is finished"

        # Check time limit
        if self.get_elapsed_time_epoch() > self.epoch_tlim + self.TLIM_GRACE_PERIOD:
            return self._fail_episode("Time exceeded")

        # Check if solution is valid
        try:
            driving_duration = tools.validate_dynamic_epoch_solution(self.epoch_instance, solution)
        except AssertionError as e:
            return self._fail_episode(e)

        if not self.is_static:
            # Mark orders of current solution as dispatched
            for route in solution:
                # Route consists of 1 indexed requests
                assert not self.request_is_dispatched[route].any()
                self.request_is_dispatched[route] = True

            # We must not have any undispatched orders that must be dispatched
            assert not (self.request_must_dispatch & ~self.request_is_dispatched).any()

        self.final_solutions[self.current_epoch] = solution
        self.final_costs[self.current_epoch] = driving_duration
        self.current_epoch += 1
        self.is_done = self.current_epoch > self.end_epoch

        observation = self._next_observation() if not self.is_done else None
        reward = -driving_duration

        self.start_time_epoch = time.time()
        return (observation, reward, self.is_done, {'error': None})

    def get_elapsed_time_epoch(self):
        assert self.start_time_epoch is not None
        return time.time() - self.start_time_epoch

    def _fail_episode(self, error):
        self.final_solutions[self.current_epoch] = None
        self.final_costs[self.current_epoch] = max(self.end_epoch - self.current_epoch, 1) * _BIG_NUMBER
        self.is_done = True
        return (None, -_BIG_NUMBER, self.is_done, {'error': str(error)})

    def _next_observation(self):
        assert not self.is_done
        assert self.start_epoch <= self.current_epoch <= self.end_epoch

        if self.is_static:
            # Static instance, don't resample requests

            self.epoch_instance = {
                'is_depot': self.instance['is_depot'],
                'customer_idx': np.arange(len(self.instance['coords'])),
                'request_idx': np.arange(len(self.instance['coords'])),
                'coords': self.instance['coords'],
                'demands': self.instance['demands'],
                'capacity': self.instance['capacity'],
                'time_windows': self.instance['time_windows'],
                'service_times': self.instance['service_times'],
                'duration_matrix': self.instance['duration_matrix'],
                'must_dispatch': ~self.instance['is_depot'],
            }
            return {
                'current_epoch': self.current_epoch,
                'current_time': 0,
                'planning_starttime': 0,
                'epoch_instance': self.epoch_instance
            }

        duration_matrix = self.instance['duration_matrix']

        # Sample new data
        current_time = self.EPOCH_DURATION * self.current_epoch
        planning_starttime = current_time + self.MARGIN_DISPATCH

        # Sample uniformly
        num_customers = len(self.instance['coords']) - 1  # Exclude depot

        # Sample data uniformly from customers (1 to num_customers)
        def sample_from_customers(k=self.MAX_REQUESTS_PER_EPOCH):
            return self.rng.integers(num_customers, size=k) + 1

        cust_idx = sample_from_customers()
        timewi_idx = sample_from_customers()
        demand_idx = sample_from_customers()
        service_t_idx = sample_from_customers()

        new_request_timewi = self.instance['time_windows'][timewi_idx]
        # Filter data that can no longer be delivered
        # Time + margin for dispatch + drive time from depot should not exceed latest arrival
        earliest_arrival = np.maximum(planning_starttime + duration_matrix[0, cust_idx], new_request_timewi[:, 0])
        # Also, return at depot in time must be feasible
        earliest_return_at_depot = earliest_arrival + self.instance['service_times'][service_t_idx] + duration_matrix[cust_idx, 0]
        is_feasible = (earliest_arrival <= new_request_timewi[:, 1]) & (earliest_return_at_depot <= self.instance['time_windows'][0, 1])

        if is_feasible.any():
            num_new_requests = is_feasible.sum()
            self.request_id = np.concatenate((self.request_id, np.arange(num_new_requests) + len(self.request_id)))
            self.request_customer_index = np.concatenate((self.request_customer_index, cust_idx[is_feasible]))
            self.request_timewi = np.concatenate((self.request_timewi, new_request_timewi[is_feasible]))
            self.request_service_t = np.concatenate((self.request_service_t, self.instance['service_times'][service_t_idx[is_feasible]]))
            self.request_demand = np.concatenate((self.request_demand, self.instance['demands'][demand_idx[is_feasible]]))
            self.request_is_dispatched = np.pad(self.request_is_dispatched, (0, num_new_requests), mode='constant')
            self.request_epoch = np.concatenate((self.request_epoch, np.full(num_new_requests, self.current_epoch)))

        # Customers must dispatch this epoch if next epoch they will be too late
        if self.current_epoch < self.end_epoch:
            earliest_arrival = np.maximum(
                planning_starttime + self.EPOCH_DURATION + duration_matrix[0, self.request_customer_index],
                self.request_timewi[:, 0]
            )
            earliest_return_at_depot = earliest_arrival + self.request_service_t + duration_matrix[self.request_customer_index, 0]
            self.request_must_dispatch = (
                (earliest_arrival > self.request_timewi[:, 1]) |
                (earliest_return_at_depot > self.instance['time_windows'][0, 1])
            )
        else:
            self.request_must_dispatch = self.request_id > 0
        # Return instance based on customers not yet dispatched
        idx_undispatched = self.request_id[~self.request_is_dispatched]
        customer_idx = self.request_customer_index[idx_undispatched]
        # Return a VRPTW instance with undispatched requests with two additional properties: customer_idx and request_idx
        time_windows = self.request_timewi[idx_undispatched]

        # Renormalize time to start at planning_starttime, and clip time windows in the past (so depot will start at 0)
        time_windows = np.clip(time_windows - planning_starttime, a_min=0, a_max=None)
        self.epoch_instance = {
            'is_depot': self.instance['is_depot'][customer_idx],
            'customer_idx': customer_idx,
            'request_idx': idx_undispatched,
            'coords': self.instance['coords'][customer_idx],
            'demands': self.request_demand[idx_undispatched],
            'capacity': self.instance['capacity'],
            'time_windows': time_windows,
            'service_times': self.request_service_t[idx_undispatched],
            'duration_matrix': self.instance['duration_matrix'][np.ix_(customer_idx, customer_idx)],
            'must_dispatch': self.request_must_dispatch[idx_undispatched],
        }
        return {
            'current_epoch': self.current_epoch,
            'current_time': current_time,
            'planning_starttime': planning_starttime,
            'epoch_instance': self.epoch_instance
        }

    def get_hindsight_problem(self):
        """After the episode is completed, this function can be used to obtain the 'hindsight problem',
        i.e. as if we had future information about all the requests.
        This includes additional info containing the release times of the requests."""
        assert self.is_done

        customer_idx = self.request_customer_index
        # Release times indicate that a route containing this request cannot dispatch before this time
        # This needs to include the margin time for the dispatch
        release_times = self.EPOCH_DURATION * self.request_epoch + self.MARGIN_DISPATCH
        release_times[self.instance['is_depot'][customer_idx]] = 0
        return {
            'is_depot': self.instance['is_depot'][customer_idx],
            'customer_idx': customer_idx,
            'request_idx': self.request_id,
            'coords': self.instance['coords'][customer_idx],
            'demands': self.request_demand,
            'capacity': self.instance['capacity'],
            'time_windows': self.request_timewi,
            'service_times': self.request_service_t,
            'duration_matrix': self.instance['duration_matrix'][np.ix_(customer_idx, customer_idx)],
            # 'must_dispatch': self.request_must_dispatch,
            'release_times': release_times
        }


import sys
import os
from sklearn.naive_bayes import BernoulliNB
from sklearn.feature_extraction.text import CountVectorizer
from nltk.corpus import stopwords
import numpy as np
from sklearn.externals import joblib
from sklearn.base import clone
from sklearn.dummy import DummyClassifier
from scipy import sparse

sys.path.insert(0, './data_files_scripts')

class Classify:
    """
    A classifier which can categorize tweets into one or more of the
    25 categories, as required by TREC.
    """
    catadictionary={'GoodsServices':0, 'SearchAndRescue':1,'InformationWanted':2,'Volunteer':3,'Donations':4,
                    'MovePeople':5, 'FirstPartyObservation': 6, 'ThirdPartyObservation': 7, 'Weather': 8,
                    'EmergingThreats': 9, 'SignificantEventChange':10, 'MultimediaShare': 11, 'ServiceAvailable': 12,
                    'Factoid': 13, 'Official': 14, 'CleanUp':15, 'Hashtags': 16, 'PastNews': 17, 'ContinuingNews': 18,
                    'Advice': 19, 'Sentiment':20, 'Discussion': 21, 'Irrelevant': 22, 'Unknown': 23, 'KnownAlready': 24}
    actionable = ['SearchAndRescue', 'MovePeople', 'InformationWanted']
    knowledge = ['FirstPartyObservation', 'EmergingThreats',
                 'SignificantEventChange', 'MultimediaShare', 'Official']

    def __init__(self, tweet_texts=None, cats=None, vocab_size=2000, model=BernoulliNB(alpha = 0.1),
        pretrained=None):
        """
        Create and train classifier. Can specify path to pretrained
        classifiers using "pretrained"

        :param tweet_texts: list of tweets to train classifier on
        :param cats: matrix of categories that belong to the tweets
        :param vocab_size: maximum number of words in classifier
        :param model: model to use for classification
        :param pretrained: path to pretained classifiers, if using
                            pretrained classifiers
        """
        self.cat = cats
        self.text = tweet_texts
        self.model = model
        self.classifiers = list()
        self.vectorizer = None
        self.vect_train = None

        if pretrained is None:
            self.train(vocab_size=vocab_size)
        else: # load pretrained classifiers
            for f in sorted(os.listdir(pretrained)):
                fn = os.fsdecode(f)
                if fn.endswith('v.pkl'):
                    self.vectorizer = joblib.load(pretrained+fn)
                else:
                    self.classifiers.append(joblib.load(pretrained+fn))

    def train(self, text=None, cats=None, vocab_size=2000):
        """
        Fits classifiers to the training data provided. If
        already trained, clears classifiers and retrains.

        :param text: tweets to train on. defaults to tweets provided
            on creation
        :param cats: actual labels
        :param vocab_size: number of words as max features
        """
        if text is None:
            text = self.text
        if cats is None:
            cats = self.cat

        # fit vectorizer
        self.vectorizer = CountVectorizer(stop_words=stopwords.words(),
                                          binary=True,
                                          max_features=vocab_size)
        self.vect_train = self.vectorizer.fit_transform(text)

        # add feature determining if hashtag in tweet
        hashtags = np.array([[1] if '#' in t else [0] for t in text])
        addtl_feat = sparse.hstack([self.vect_train, hashtags])

        # clear classifiers (in case of retraining)
        self.classifiers = list()

        cat_arr = np.array(cats) # convert to array

        # train
        for i in range(0, len(cat_arr[0])):
            # unknown should not be predicted unless no other category found
            # so use dummy classifier only predicting 0 for unknown
            if i == self.catadictionary['Unknown']:
                m = DummyClassifier('constant', constant=0)
            else:
                # clone original model. "clone" is needed or it will
                # just keep training the original model on all categories
                m = clone(self.model)
            # fit model and append to list of models
            c = m.fit(addtl_feat, cat_arr[:,i])
            self.classifiers.append(c)

    def save_classifier(self, path='pretrained/'):
        """
        Saves the classifier to the specified path

        :param path: directory to save classifier and vectorizer into
        """
        # save vectorizer
        joblib.dump(self.vectorizer, path+'v.pkl', compress=1)
        # save classifiers
        for n in range(0,len(self.classifiers)):
            joblib.dump(self.classifiers[n], path+'c%02d.pkl' % n, compress=1)
        print("Classifiers saved to: " + path)

    def map_id(self, category):
        """
        Return the indices of the given categories e.g. 0 = 'Advice'

        :param category: categories to lookup

        :return: list of int index
        """
        returner = []
        for c in category:
            if c in self.catadictionary:
                returner.append(self.catadictionary.get(c))
        return returner

    def mat_all_categories(self, actual, prediction):
        """
        Evaluator returning confusion matrices for all categories
        as a dictionary, with the keys being the categories and
        values being confusion matrix dictionaries

        :param actual: actual category matrix
        :param prediction: prediction matrix

        :return: dictionary of confusion matrix by category
        """
        ind_to_cat = self.category_indices()
        ret = dict()
        actual_arr = np.array(actual)
        prediction_arr = np.array(prediction)
        for n in range(0, len(actual[0])):
            vals = self.mat_one_category(actual_arr[:,n], prediction_arr[:,n])
            ret[ind_to_cat[n]] = vals
        return ret

    def mat_one_category(self, actual, prediction):
        """
        Evaluator returning confusion matrix for a single category as a dict

        :param actual: numpy array of binary category vals
        :param prediction: binary numpy array of predictions

        :return: dict of number of predictions,
                true pos, true neg, false pos,
                and false neg for predictions
        """
        eval = {'Number of Predictions': len(actual),
                'True Positive': 0, 'True Negative': 0,
                'False Positive': 0, 'False Negative': 0}

        for x in range(0, len(actual)):
            if actual[x] == 1:
                if prediction[x] == 1:
                    eval['True Positive'] += 1
                else:
                    eval['False Negative'] +=1
            else:
                if prediction[x] == 1:
                    eval['False Positive'] += 1
                else:
                    eval['True Negative'] += 1
        return eval

    def evaluate(self, actual, prediction):
        """
        Evaluator, returning overall confusion matrix, accuracy
        recall, precision, f1, one label, category, and perfect match
        scores as a dictionary

        :param actual: a matrix of actual binary values
        :param prediction: a matrix of the predicted binary values

        :return: dictionary of number of predictions, true/false
                positive/negative, number of rows with at least 1
                true positive, number of perfect match rows,
                accuracy, precision, recall, f1, one label,
                and perfect match scores
        """
        # result matrix
        evaluation_mat = {'Number of Predictions': len(actual)*len(actual[0]),
                          'True Positive': 0, 'True Negative': 0,
                          'False Positive': 0, 'False Negative': 0,
                          'One Label': 0, 'Perfect Match': 0}
        one_label = False # used to check if at least one label in current row is true pos

        # evaluate by row
        for x in range(0, len(actual)):
            # check for perfect match
            if np.array_equal(actual[x], prediction[x]):
                evaluation_mat['Perfect Match'] += 1

            # iterate through predictions
            for y in range(0, len(actual[x])):
                if actual[x][y] == 1:
                    if prediction[x][y] == 1:
                        # both positive
                        evaluation_mat['True Positive'] += 1
                        one_label = True
                    else:
                        # prediction negative, actually positive
                        evaluation_mat['False Negative'] += 1
                else:
                    if prediction[x][y] == 1:
                        # prediction positive, actually negative
                        evaluation_mat['False Positive'] += 1
                    else:
                        # both negative
                        evaluation_mat['True Negative'] += 1

            # check if at least one true pos
            if one_label:
                evaluation_mat['One Label'] += 1
            one_label = False

        # once all rows checked, use confusion matrix and one label /
        # perfect match counts to calculate other statistics
        stats = self.stats_calc(evaluation_mat['True Positive'],
                                evaluation_mat['True Negative'],
                                evaluation_mat['False Positive'],
                                evaluation_mat['False Negative'],
                                evaluation_mat['One Label'],
                                evaluation_mat['Perfect Match'],
                                cats=len(actual[0]))
        ret = {**evaluation_mat, **stats}
        return ret

    def predict(self, tweets):
        """
        Returns an array of predictions for the given features.

        :param tweets: a list or array of string tweets
        :returns: predictions matrix

        :throws RuntimeError: if classifiers have not been trained
        """
        if len(self.classifiers) == 0:
            raise RuntimeError("Classifiers have not been trained!")
        tokenized = self.vectorizer.transform(tweets)
        hashtags = [[1] if '#' in t else [0] for t in tweets]
        addtl_feat = sparse.hstack([tokenized, hashtags])

        predictions = np.zeros((len(tweets), len(self.classifiers)))

        for i in range(0, len(self.classifiers)):
            predictions[:, i] = self.classifiers[i].predict(addtl_feat)

        # if nothing predicted, category should be unknown
        for row in predictions:
            if np.sum(row) == 0:
                row[self.catadictionary['Unknown']] = 1
        return predictions

    def stats_calc(self, tp, tn, fp, fn, one_lab, perf_match, cats=25):
        """
        Calculate summary statistics, return as dict.

        :param tp: # of true postitives
        :param tn: # of true negatives
        :param fp: # of false positives
        :param fn: # of false negatives
        :param one_lab: # with 1 true positive
        :param perf_match: # with all labels correct

        :return: dict of scores
        """
        ret = dict()
        n = tp + tn + fp + fn

        # one label/perfect match is out of number of
        # tweets, not predictions
        t = n/cats
        ret['One Label Score'] = one_lab/t
        ret['Perfect Match Score'] = perf_match/t
        ret['Accuracy'] = (tp+tn)/n
        try:
            ret['Precision'] = tp/(tp+fp)
        except ZeroDivisionError:
            ret['Precision'] = 0
        try:
            ret['Recall'] = tp/(tp+fn)
        except ZeroDivisionError:
            ret['Recall'] = 0
        try:
            ret['F1 Score'] = (2*(ret['Precision']*ret['Recall']))/(ret['Precision']+ret['Recall'])
        except ZeroDivisionError:
            ret['F1 Score'] = 0
        return ret

    def category_indices(self):
        """
        Return dict of index to category, based on the catadict
        """
        ret = dict()
        for key in self.catadictionary:
            ret[self.catadictionary[key]] = key
        return ret

    def special_mats(self, mat):
        """
        Calculate knowledge and actionable confusion matrices from a category confusion matrix dict

        :param mat: a dictionary of conf matrices for categories as formed by self.mat_all_categories

        :return: dictionary of name to dict confusion matrix

        :raise ValueError: if any of the stats in self.actionable or self.knowledge don't appear in mat
        """
        act = {'Number of Predictions': 0,
               'True Positive': 0, 'True Negative': 0,
               'False Positive': 0, 'False Negative': 0}
        know = {'Number of Predictions': 0,
                'True Positive': 0, 'True Negative': 0,
                'False Positive': 0, 'False Negative': 0}
        for m in self.actionable:
            if m in mat:
                for stat in mat[m]:
                    act[stat] += mat[m][stat]
            else:
                raise ValueError("Incorrectly formed dictionary passed to calculate actionable stats.")
        for m in self.knowledge:
            if m in mat:
                for stat in mat[m]:
                    know[stat] += mat[m][stat]
            else:
                raise ValueError("Incorrectly formed dictionary passed to calculate knowledge stats.")
        return {'Actionable': act, 'Knowledge': know}


#!/usr/bin/python3
import numpy as np
import sys
from astropy import units as u
from skyfield.api import load
from skyfield.api import Topos

class shadow_calc(object):
    def __init__(self, observatory_name="LCO",
    observatory_elevation=2380.0*u.m,
    observatory_lat='29.0146S',
    observatory_lon = '70.6926W',
    jd=2459458,
    eph=None,
    earth=None,
    sun=None):
        """
        Initialization sets a default observatory to LCO, and the default time to the date when initialized.
        """
        super().__init__()

        # Load the ephemeral datat for the earth and sun.
        if eph is None:
            self.eph = load('de421.bsp')

        # Get functions for the earth, sun and observatory
        self.earth = earth
        if self.earth is None:
            self.earth = self.eph['earth']

        self.sun = sun
        if self.sun is None:
            self.sun = self.eph['sun']

        self.observatory_elevation = observatory_elevation
        try:
            self.observatory_elevation.to("m")
        except:
            sys.exit("Observatory elevation does not have unit of length")

        self.observatory_name = observatory_name
        self.observatory_lat = observatory_lat
        self.observatory_lon = observatory_lon
        self.observatory_topo = Topos(observatory_lat, observatory_lon, elevation_m=self.observatory_elevation.to("m").value)
        self.ts = load.timescale()

        """
        These are going to be vectors which are the x,y,z positions of the puzzle.
        Credit for this solution to the intersection goes to:  Julien Guertault @ http://lousodrome.net/blog/light/2017/01/03/intersection-of-a-ray-and-a-cone/
        """
        self.ra = None
        self.dec = None
        self.pointing_unit_vectors = None
        self.xyz_earth = None
        self.xyz_sun = None
        self.xyz_observatory = None
        self.v = None
        self.xyz_c = None
        self.a = None
        self.b = None
        self.c = None
        self.delta = None
        self.dist = None
        self.heights = None

        # Those are very self explanitory
        self.earth_radius=6.357e6*u.m
        self.sun_radius=695.700e6*u.m

        # Distance from Sun to the earth.
        self.d_se = 1*u.au

        # Distance from tip of the shadow cone to the earth
        self.d_ec = self.d_se * (self.earth_radius / self.sun_radius)

        # Opening angle of the shadow cone
        self.shadow_cone_theta  = np.arctan(self.earth_radius/self.d_ec)
        self.shadow_cone_cos_theta_sqr = np.square(np.cos(self.shadow_cone_theta))

        # Finally, updated the xyz coordinate vectors of earth, sun and observatory.
        self.jd = jd
        if jd is not None:
            self.t = self.ts.tt_jd(jd)
        else:
            self.t = self.ts.now()

        self.update_time(jd)

    def cone_ra_dec(self):
        # This returns the cone RA,Dec for testing.
        # It has a shadow height that is analyically easy to calculate
        observatory_to_c = self.xyz_c - self.xyz_observatory

        # Leave dec in steradian for just a moment
        dec = np.arctan(observatory_to_c[2]/np.sqrt(np.square(observatory_to_c[0])+np.square(observatory_to_c[1])))

        # Use dec in steradians to solve for ra.
        ra = np.degrees(np.arccos(observatory_to_c[0]/ (self.vecmag(observatory_to_c) * np.cos(dec))))

        #Now convert to degree
        dec = np.degrees(dec)

        return ra, dec

    def set_coordinates(self, ra, dec):
        # Set coordinates and calculate unit vectors.
        ra = np.atleast_1d(ra)
        dec = np.atleast_1d(dec)
        self.ra = ra
        self.dec = dec
        self.pointing_unit_vectors = np.zeros(shape=(len(self.ra),3))

        a = np.pi / 180.
        self.pointing_unit_vectors[:,0] = np.cos(self.ra*a)*np.cos(self.dec*a)
        self.pointing_unit_vectors[:,1] = np.sin(self.ra*a)*np.cos(self.dec*a)
        self.pointing_unit_vectors[:,2] = np.sin(self.dec*a)

    def update_time(self, jd=None):
        """ Update the time, and all time dependent vectors """
        if jd is not None:
            self.jd = jd
        self.t = self.ts.tt_jd(self.jd)
        self.update_xyz()
        self.update_vec_c()
        self.co = self.xyz_c - self.xyz_observatory

    def update_xyz(self):
        """
        Due to the nature of the vectors, it is not possible to combine the earth and the observatory topo directly. Therefore
        this requires the calculation of an observatory position to be done using the XYZ positions defined at a particular time, and with a particular unit.
        To prevent unit issues later from arising SI units are adopted. In the case where other units are desired, like plotting the solar system, local conversions
        can be done.
        """
        self.xyz_earth = self.earth.at(self.t).position.au
        self.xyz_sun = self.sun.at(self.t).position.au
        self.xyz_observatory = self.earth.at(self.t).position.au + self.observatory_topo.at(self.t).position.au

    def update_vec_c(self):
        """
        d_se : distance sun to earth
        d_ec : distance earth to (c)one tip
        c_unit_vec: unit vector from cone to earth, or earth to sun.

        """
        # We reverse the vector sun to earth explicitly with -1.0 for clarity
        self.v = -1.0*(self.xyz_earth - self.xyz_sun)/np.sqrt(np.sum(np.square(self.xyz_earth-self.xyz_sun)))
        self.xyz_c = self.xyz_earth - self.v * self.d_ec.to("au").value

    def solve_for_height(self, mask=None, unit="km"):
        if mask is None:
            mask = np.full(len(self.pointing_unit_vectors), True)

        puv = self.pointing_unit_vectors[mask]

        self.a = np.square(np.sum(puv*self.v,axis=1)) -  self.shadow_cone_cos_theta_sqr
        self.b = 2*( np.sum(puv*self.v, axis=1) * np.sum(self.co*self.v) - np.sum(puv*self.co, axis=1)*self.shadow_cone_cos_theta_sqr)
        self.c = np.square(np.sum(self.co*self.v)) - np.sum(self.co * self.co) * self.shadow_cone_cos_theta_sqr
        self.delta = np.square(self.b) - 4 * self.a * self.c

        self.dist = np.full(len(self.a), np.nan)
        dist_b1 = np.full(len(self.a), np.nan)
        dist_b2 = np.full(len(self.a), np.nan)

        # Get P distance to point.
        positive_delta = self.delta >= 0
        dist_b1[positive_delta] = -1*(-self.b[positive_delta]+np.sqrt(self.delta[positive_delta])) / (2*self.a[positive_delta])
        dist_b2[positive_delta] = -1*(-self.b[positive_delta]-np.sqrt(self.delta[positive_delta])) / (2*self.a[positive_delta])

        dist_b1[dist_b1 < 0.0] = np.nan
        dist_b2[dist_b2 < 0.0] = np.nan

        self.dist[positive_delta] = np.nanmin([dist_b1[positive_delta], dist_b2[positive_delta]], axis=0)

        # caluclate xyz of each ra, using the distance to the shadow intersection and the normal vector form the observatory
        pointing_xyz = puv * self.dist[:,np.newaxis] + self.xyz_observatory

        self.heights = (self.vecmag(pointing_xyz - self.xyz_earth)*u.au - self.earth_radius).to(unit).value

    def get_heights(self, mask=None, jd=None, return_heights=True,unit=u.km):
        if jd is not None:
            self.jd = jd
            self.update_time()
        self.solve_for_height(mask=mask, unit=unit)
        if return_heights:
            return self.heights

    def vecmag(self, a):
        """ Return the magnitude of a set of vectors around an abritrary origin """
        if len(np.shape(a)) == 1:
            return np.sqrt(np.square(a[0]) + np.square(a[1]) + np.square(a[2]))
        else:
            return np.sqrt(np.square(a[:, 0]) + np.square(a[:, 1]) + np.square(a[:, 2]))

class orbit_animation(object):
    def __init__(self, calculator, jd0=None, djd=1/24.):
        if jd0 is None:
            self.jd0 = calculator.jd
        else:
            self.jd0 = jd0
        self.djd = djd

        self.calculator = calculator
        self.calculator.observatory_zenith_topo = Topos(calculator.observatory_lat, calculator.observatory_lon, elevation_m=self.calculator.earth_radius.to("m").value*10)

        import matplotlib.pyplot as plt
        import matplotlib.animation as animation
        import matplotlib.patches as patches

        self.plt = plt
        self.animation = animation
        self.patches = patches
        self.fig, self.ax = self.plt.subplots(figsize=(50,50))

    def init_plotting_animation(self):
        self.line1, = self.ax.plot([], [], 'ro', lw=2)
        self.line2, = self.ax.plot([], [], '.-', lw=5)
        self.line3, = self.ax.plot([], [], '.-', lw=5)
        self.path, = self.ax.plot([],[],'-', lw=1)

        self.ax.grid()
        self.xdata, self.ydata = [], []
        self.pathx, self.pathy = [], []

        self.ax.set_ylim(-150 * self.calculator.earth_radius.to("au").value, 150 * self.calculator.earth_radius.to("au").value)
        self.ax.set_xlim(-150 * self.calculator.earth_radius.to("au").value, 150 * self.calculator.earth_radius.to("au").value)
        del self.xdata[:]
        del self.ydata[:]
        self.line1.set_data(self.xdata, self.ydata)
        self.line2.set_data(self.xdata, self.ydata)

    def do_animation(self, N_days=1.0):
        ani = self.animation.FuncAnimation(self.fig, self.animation_update_positions, frames=(24*N_days), repeat_delay=0,
                                        blit=False, interval=100, init_func=self.init_plotting_animation, repeat=0)
        ani.save("/tmp/im.mp4")

    def snap_shot(self,jd,ra,dec, show=True, extra=False):
        self.init_plotting_animation()
        self.calculator.set_coordinates(np.array([ra]),np.array([dec]))

        self.calculator.update_time(jd)

        self.calculator.xyz_observatory_zenith = self.calculator.xyz_earth + self.calculator.observatory_zenith_topo.at(self.calculator.t).position.au

        height_au = 3.0 * self.calculator.earth_radius.to("au").value

        self.ax.set_xlim( ( self.calculator.xyz_earth[0] - height_au * 160, self.calculator.xyz_earth[0] + height_au * 160 ) )
        self.ax.set_ylim( ( self.calculator.xyz_earth[1] - height_au * 90, self.calculator.xyz_earth[1] + height_au * 90 ) )

        circ = self.patches.Circle((self.calculator.xyz_earth[0], self.calculator.xyz_earth[1]), self.calculator.earth_radius.to( "au" ).value, alpha=0.8, fc='yellow')
        self.ax.add_patch( circ )

        los_xyz =  self.calculator.xyz_observatory + self.calculator.d_ec.to("au").value * self.calculator.pointing_unit_vectors[0]

        #self.line1.set_data( (x_heights * u.m).to( "au" ).value, ( y_heights * u.m ).to("au").value )
        self.line1.set_data( [ self.calculator.xyz_c[0] ], [ self.calculator.xyz_c[1] ] )
        self.line2.set_data( [ self.calculator.xyz_observatory_zenith[0] ], [ self.calculator.xyz_observatory_zenith[1] ] )
        self.line3.set_data( [ self.calculator.xyz_observatory[0] ] , [ self.calculator.xyz_observatory[1] ] )
        self.ax.plot([self.calculator.xyz_c[0], self.calculator.xyz_sun[0]],[self.calculator.xyz_c[1], self.calculator.xyz_sun[1]], alpha=0.5)
        self.ax.plot([los_xyz[0], self.calculator.xyz_observatory[0]],[los_xyz[1], self.calculator.xyz_observatory[1]], c="k", alpha=0.5)
        if extra is not False:
            #This will plot a line using arrays provided in the extra bin.
            self.ax.plot(extra[0], extra[1])

        self.plt.gca().set_aspect('equal', adjustable='box')
        if show:
            self.plt.show()
        else:
            return self.plt

    def animation_update_positions(self, frame):
        if len( self.ax.patches ) > 1:
            del( self.ax.patches[-1] )

        jd = self.jd0 + frame*self.djd

        #self.calculator.t = self.calculator.ts.tt_jd( self.jd0 + frame*self.djd)
        self.calculator.update_time(jd)

        self.calculator.xyz_observatory_zenith = self.calculator.xyz_earth + self.calculator.observatory_zenith_topo.at(self.calculator.t).position.au

        height_au = 3.0 * self.calculator.earth_radius.to("au").value

        ######################################################################
        # The next set of lines calculates shadow heights. That's for later  #
        ######################################################################
        # N_ra = 24
        # N_dec = 9
        # x_heights = np.zeros(len(N_ra * N_dec)+1)
        # y_heights = np.zeros(len(N_ra * N_dec)+1)
        #
        # for ra_i, ra in enumerate(np.linspace(1,24,N_ra)):
        #     for dec_j, dec in enumerate(np.linspace(-90, 0, num=N_dec, endpoint=True)):
        #         # for height in heights:
        #             self.calculator.point_along_ray =  position_from_radec(ra, dec, distance=height_au, epoch=None, t=self.calculator.t, center=self.calculator.observatory_topo, target=None) #, observer_data=LCO_topo.at(t).observer_data)
        #             self.calculator.xyz_along_ray = self.calculator.xyz_observatory + self.calculator.point_along_ray.position.au
        #             x_heights[ ra_i + dec_j*N_ra ] = self.calculator.xyz_along_ray[0]
        #             y_heights[ ra_i + dec_j*N_ra ] = self.calculator.xyz_along_ray[1]
        #
        # x_heights[-1] = self.calculator.xyz_earth[0]
        # y_heights[-1] = self.calculator.xyz_earth[1]
        ######################################################################
        self.ax.set_xlim( ( self.calculator.xyz_earth[0] - height_au * 150, self.calculator.xyz_earth[0] + height_au * 150 ) )
        self.ax.set_ylim( ( self.calculator.xyz_earth[1] - height_au * 150, self.calculator.xyz_earth[1] + height_au * 150 ) )

        circ = self.patches.Circle((self.calculator.xyz_earth[0], self.calculator.xyz_earth[1]), self.calculator.earth_radius.to( "au" ).value, alpha=0.8, fc='yellow')
        self.ax.add_patch( circ )

        #self.line1.set_data( (x_heights * u.m).to( "au" ).value, ( y_heights * u.m ).to("au").value )
        self.line1.set_data( [ self.calculator.xyz_c[0] ], [ self.calculator.xyz_c[1] ] )
        self.line2.set_data( [ self.calculator.xyz_observatory_zenith[0] ], [ self.calculator.xyz_observatory_zenith[1] ] )
        self.line3.set_data( [ self.calculator.xyz_observatory[0] ] , [ self.calculator.xyz_observatory[1] ] )
        self.pathx.append( self.calculator.xyz_earth[0] )
        self.pathy.append( self.calculator.xyz_earth[1] )
        self.path.set_data( self.pathx, self.pathy )

        #line2.set_data([sun.at(t).position.au[0]],[sun.at(t).position.au[1]])

def test_shadow_calc():
    jd = 2459458.5+20 + (4.75)/24.
    # Initiate tests.
    test_results ={}

    try:
        test = "Create Class"
        calculator = shadow_calc()
        test_results[test] = "Pass"
    except:
        test_results[test] = "Fail"

    orbit_ani = orbit_animation(calculator)
    orbit_ani.calculator.update_time(jd)
    ra, dec = orbit_ani.calculator.cone_ra_dec()

    orbit_ani.snap_shot(jd=jd, ra=ra, dec=dec)
    # orbit_ani.do_animation()

    compare_old = True
    if compare_old:
        eph = load('de421.bsp')
        import lvmsurveysim.utils.iterative_shadow_height_lib as iterative_shadow_height_lib
        iter_calc = iterative_shadow_height_lib.shadow_calc(observatory_name='LCO',
                        observatory_elevation=2380*u.m,
                        observatory_lat='29.0146S', observatory_lon='70.6926W',
                        eph=eph, earth=eph['earth'], sun=eph['sun'])
        iter_calc.update_t(jd)
        old_h = iter_calc.height_from_radec(ra/15., dec, simple_output=True)['height']
        new_h = calculator.get_heights(return_heights=True, unit="m")
        print("old: %f; new: %f"%(old_h, new_h))

    from astropy.coordinates import SkyCoord
    from astropy.coordinates import Angle, Latitude, Longitude
    from astropy import units as u

    try:
        test = "Shadow RA/DEC"
        ra, dec = calculator.cone_ra_dec()
    except:
        test_results[test] = "Fail"

    try:
        test = "Set Coordinates"
        calculator.set_coordinates(np.array([ra]), np.array([dec]) )
        test_results[test] = "Pass"
    except:
        test_results[test] = "Fail"

    try:
        test = "set jd=2459458.5"
        jd = 2459458.5+20 + 4.75/24.
        calculator.update_time(jd)
        test_results[test] = "Pass"
    except:
        test_results[test] = "Fail"

    try:
        test = "Run"
        calculator.get_heights()
        test_results[test] = "Pass"
    except:
        test_results[test] = "Fail"

    try:
        test = "loop"
        for jd in np.linspace(jd, jd+1, 1/24.):
            calculator.update_time(jd)
            heights = calculator.get_heights(return_heights=True, unit="km")
            #print("height_v(jd=%f) = %f"%(jd, heights[0]))
        test_results[test] = "Pass"
    except:
        test_results[test] = "Fail"

    for test in test_results.keys():
        print ("%s -> %s"%(test, test_results[test]))

if __name__ == "__main__":
    test_shadow_calc()

{'plt': 'matplotlib.pyplot', 'animation': 'matplotlib.animation', 'patches': 'matplotlib.patches'}


#!/usr/bin/python
#-*- coding: utf-8 -*-

import sys
import time
import urllib
import urllib.parse
import requests
import numpy as np
import imp
from bs4 import BeautifulSoup
from openpyxl import Workbook
from urllib import request
from urllib import error
from fake_useragent import UserAgent
imp.reload(sys)

# Fake User Agents
ua = UserAgent()
people=[['No.','Name','Company']]

def builder_spider():
    page_num=1;
    builder_list=[['No.','Name','Reg No','Areas','Contact','T','F','Address','Web','Key Projects','Awards']]

    while(page_num>0):
        url='https://www.mbawa.com/become-a-member/find-a-member/page/'+str(page_num)+'/?member_type=residential-builder&button&find-a-member=true'
        time.sleep(np.random.rand()*5) # Random waitling time to cover

        # Try to get html code
        try:
            req = request.Request(url, headers={ua.random})
            source_code = request.urlopen(req).read()
            plain_text=str(source_code).replace('
',' ').replace('
',' ').replace('
',' ')
        except (error.HTTPError, error.URLError) as e:
            print_builder_lists_excel(builder_list,page_num)
            print ('Access denied.')
            continue

        soup = BeautifulSoup(plain_text)
        list_soup = soup.find('ol', {'class': 'results table-list'})

        if list_soup == None:
            page_num=-1
            print_builder_lists_excel(builder_list,page_num)
            print('All result has been stored.')
            break

        for builder_info in list_soup.findAll('article'):
            name = builder_info.find('a', {'target':'_blank'}).string.strip()
            try:
                company_url = builder_info.find('a', {'target':'_blank'}).get('href')
                detail = get_detail(company_url, name)
            except:
                print('No detail get')
                detail=[''] * 9
            builder_list.append([len(builder_list),name,detail[0],detail[1],detail[2],detail[3],detail[4],detail[5],detail[6],detail[7],detail[8]])
        print ('Downloading Information From Page %d' % page_num)
        page_num+=1
    return 1

def get_detail(url, name):
    detail=[''] * 9
    global people

    try:
        req = request.Request(url, headers=hds[np.random.randint(0,len(hds))])
        source_code = request.urlopen(req).read()
        plain_text=str(source_code).replace('
','').replace('
','').replace('
','').replace('\\n','')
    except (error.HTTPError, error.URLError) as e:
        print_builder_lists_excel(builder_list,0)
        print ('Access denied.')
        print (e)

    soup = BeautifulSoup(plain_text)

    try:
        ul = soup.find('ul', {'class':'member-meta'}).find_all('li')
    except:
        print('Company data not found: '+name)
        return detail

    for li in ul:
        if(li.find('strong') == None):
            try:
                [s.extract() for s in li('strong')]
                content = li.string.strip()
            except:
                print('Wrong type address')
                continue
            else:
                detail[5]=content
        elif(li.find('strong').string == 'Builders Reg No: '):
            try:
                [s.extract() for s in li('strong')]
                content = li.string.strip()
            except:
                print('Wrong type Reg No.')
                continue
            else:
                detail[0]=content
        elif(li.find('strong').string == 'Areas:'):
            try:
                [s.extract() for s in li('strong')]
                content = li.string.strip()
            except:
                print('Wrong type area')
                continue
            else:
                detail[1]=content
        elif(li.find('strong').string == 'Contact:'):
            try:
                [s.extract() for s in li('strong')]
                content = li.string.strip()
            except:
                print('Wrong type contact')
                continue
            else:
                if content != '':
                    people.append([content,name])
                    detail[2]=content
        elif(li.find('strong').string == 'T:'):
            try:
                [s.extract() for s in li('strong')]
                content = li.string.strip()
            except:
                print('Wrong type T')
                continue
            else:
                detail[3]=content
        elif(li.find('strong').string == 'F:'):
            try:
                [s.extract() for s in li('strong')]
                content = li.string.strip()
            except:
                print('Wrong type F')
                continue
            else:
                detail[4]=content
    try:
        web = soup.find('a', {'class':'btn btn-primary btn-uppercase m-b-1'}).get('href')
        detail[6]=web
    except:
        print('No web')

    divl = soup.find_all('div', {'class':'m-b-3'})
    for item in divl:
        if(item.find('h3') != None and item.find('h3').string == 'Key Projects'):
            item_list=[]
            li=item.find_all(['li','p'])
            for i in li:
                try:
                    if i.string.strip() != None:
                        item_list.append(i.string.strip())
                except:
                    continue
            detail[7]=','.join(item_list)
        elif(item.find('h3') != None and item.find('h3').string == 'Awards'):
            item_list=[]
            li=item.find_all(['li','p'])
            for i in li:
                try:
                    if i.string.strip() != None:
                        item_list.append(i.string.strip())
                except:
                    continue
            detail[8]=','.join(item_list)

    stuff = soup.find('h4', {'class':'primary'})
    if stuff != None and len(stuff.find_next_siblings('ul')) != 0:
        for person in stuff.find_next_siblings('ul')[0].find_all('li'):
            try:
                if person.string.strip() != '':
                    people.append([len(people),person.string.strip(),name])
            except:
                continue
    elif stuff != None and stuff.find_next_siblings('p') != None:
        for person in stuff.find_next_siblings('p'):
            try:
                if person.string.strip() != '':
                    people.append([len(people),person.string.strip(),name])
            except:
                continue

    return detail

def print_builder_lists_excel(builder_list,page_num):
    wb=Workbook(write_only=True)
    global people
    people = list(set([tuple(t) for t in people]))

    ws=wb.create_sheet('builder')
    ws2=wb.create_sheet('people')
    for bl in builder_list:
       ws.append([bl[0],bl[1],bl[2],bl[3],bl[4],bl[5],bl[6],bl[7],bl[8],bl[9],bl[10]])
    for person in people:
       ws2.append([person[0],person[1],person[2]])
    save_path='builder list '+time.strftime("%Y-%m-%d", time.localtime())+' page '+str(page_num)+'.xlsx'
    wb.save(save_path)

if __name__=='__main__':
    builder_lists=builder_spider()


# version: 0.0.1

# import logging
import msvcrt
import os
from datetime import datetime, timedelta, time
import sys
from pydub import AudioSegment, effects
import shutil

def resource_path(relative_path):
    try:
        base_path = sys._MEIPASS
    except Exception:
        base_path = os.path.abspath(".")

    return os.path.join(base_path,
                        relative_path)

# Returns dictionary {<datetime: created> : <str: file name>} of voice files
def get_voice_files():
    dict_voice_files = {}
    all_files = os.listdir()

    for file in all_files:
        if file.startswith('audioclip') and file.endswith('.mp4'):
            try:
                file_timestamp = file[9:19]
                file_date = datetime.fromtimestamp(int(file_timestamp))
                dict_voice_files[file_date] = file

                # for debug
                # logging.info("Detected file -> created: {} | file name: {}".format(file_date.strftime("%d/%m/%Y, %H:%M:%S"), file))

            except (ValueError, TypeError):
                print("Error while getting int timestamp from str | file name: {}".format(file))
                break

    return dict_voice_files

def group_files_by_date(ungrouped_files, date_range):
    grouped_files = {}

    for date in ungrouped_files:
        group_date = datetime.date(date)
        if datetime.time(date) < time(5, 0, 0):
            group_date = group_date - timedelta(days=1)

        if date_range[0] <= group_date <= date_range[1]:
            if group_date in grouped_files.keys():
                grouped_files[group_date][date] = ungrouped_files[date]
            else:
                grouped_files[group_date] = {date : ungrouped_files[date]}

    return grouped_files

# Print iterations progress
def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 50, fill = '▒', printEnd = "\r"):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()

def process_audio_files(grouped_files):
    printProgressBar(0,
                    len(grouped_files.keys()))

    if os.path.isdir("export"):
        shutil.rmtree("export")
    os.mkdir("export")

    for group_key in grouped_files:
        to_be_merged = AudioSegment.empty()

        for file_key in grouped_files[group_key]:
            to_be_merged += AudioSegment.from_file(grouped_files[group_key][file_key],
                                                format="mp4")

        to_be_merged += AudioSegment.from_file(resource_path("beep.wav"), format="wav")

        try:
            normalized_audio = effects.normalize(to_be_merged)
            normalized_audio.export("export/{}.mp3".format(group_key.strftime("%Y-%m-%d")), format="mp3")
        except Exception as e:
            print(e)
            msvcrt.getch()

        printProgressBar(list(grouped_files).index(group_key) + 1,
                            len(grouped_files.keys()))

    print("\nDONE | Your merged voice messages are located in 'export' directory!")

def is_valid_range_date_format(str_date):
    try:
        datetime.strptime(str_date, '%Y.%m.%d')
        return True
    except ValueError:
        return False

# Get date range from user
def get_process_range():
    range = [datetime(1900, 1, 1).date(), datetime(3000, 1, 1).date()]

    range_from_str = input("Process messages from [yyyy.mm.dd] (empty for no limit): ")
    while range_from_str != '' and not is_valid_range_date_format(range_from_str):
        range_from_str = input("typo ► Process messages from [yyyy.mm.dd] (empty for no limit): ")

    if range_from_str != '':
        range[0] = datetime.strptime(range_from_str, '%Y.%m.%d').date()

    range_to_str = input("Process messages to [yyyy.mm.dd] (empty for no limit): ")
    while range_to_str != '' and not is_valid_range_date_format(range_to_str):
        range_to_str = input("typo ► Process messages to [yyyy.mm.dd] (empty for no limit): ")

    if range_to_str != '':
        range[1] = datetime.strptime(range_to_str, '%Y.%m.%d').date()

    return range

def main(args=None):
    print("                 ╔════════════════════════════════╗")
    print("                 ║          Voicetelling          ║")
    print("                 ╚════════════════════════════════╝")
    print("This script merges Facebook Messenger voice messages by day into .mp3 files.")
    print("► https://github.com/smolikja/voicetelling")
    print("Developed by smolikja: https://github.com/smolikja")
    print("==================================================")

    # # for debug
    # logging.basicConfig(filename="std.log",
    #                     format='%(asctime)s %(message)s',
    #                     filemode='w',
    #                     level= logging.DEBUG)

    # Files into dictionary {<datetime: created> : <str: file name>}
    dict_voice_files = get_voice_files()

    if bool(dict_voice_files):
        range = get_process_range()

        # Files into dictionary {<datetime: group time> : {<datetime: created> : <str: file name>}}
        dict_grouped_files = group_files_by_date(dict_voice_files, date_range= range)

        print("\nProcessing into {} audio files...".format(len(dict_grouped_files.keys())))

        # Merge and export audio files
        process_audio_files(dict_grouped_files)
    else:
        print("\n>>> No voice files found. <<<\n")

    print("Press any key to exit.")
    msvcrt.getch()

if __name__ == '__main__':
    main()


from typing import List

class Solution:
    def threeSumClosest(self, nums: List[int], target: int) -> int:
        n=len(nums)
        nums.sort()
        res=nums[0]+nums[1]+nums[2]
        for i in range(n):
            l,r=i+1,n-1
            while(l<r):
                sum=nums[i] + nums[l] + nums[r]
                if abs(sum-target)<abs(res-target):
                    res = sum
                if(res-target>0):
                    r-=1
                elif(res-target<0):
                    l+=1
                else:
                    return res
        return res

if __name__ == '__main__':
    nums=[-1,2,1,4]
    target=10
    print(Solution().threeSumClosest(nums,target))


from flask import render_template
from flask import request
from flask import Flask

from replit import db
import datetime
import time

app = Flask(__name__)

@app.route('/')
def index():
  return render_template('index.html')

@app.route('/history', methods=['GET', 'POST'])
def history():
  if request.method == 'GET':
    return render_template('history.html', data=db['history'])
  else:
    db['history'] = request.json['history']
    return {'success': True}

@app.route('/sync')
def sync():
  if time.time()-db['last_synced'] > 10:
    return {
      'internet': False,
      'last_synced': [round(float(i)) for i in str(datetime.timedelta(seconds=time.time()-db['last_synced'])).split(':')]
    }
  else:
    return {
      'internet': True,
      'last_synced': round(time.time()-db['last_synced'])
    }

@app.route('/update', methods=['POST'])
def update():
  if request.headers.get('x-forwarded-for') == 'YOUR IP':
    db['last_synced'] = float(request.form['last_synced'])

  return {'success': True}

# db['last_synced'] = time.time()
# db['history'] = []
app.run(host='0.0.0.0', port=8080)


import sys
from PyQt5.QtWidgets import QApplication, QMainWindow, QTreeView
from PyQt5.Qt import QStandardItemModel, QStandardItem
from PyQt5.QtGui import QFont, QColor

class StandardItem(QStandardItem):
    def __init__(self, txt = "", fontSize = 12, setBold = False, color = QColor(0, 0, 0)):
        super().__init__()
        fnt = QFont('Open Sans', fontSize)
        fnt.setBold(setBold)
        self.setEditable(False)
        self.setForeground(color)
        self.setFont(fnt)
        self.setText(txt)

class AppDemo(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("World diagram")
        self.resize(500, 700)
        treeView = QTreeView()
        treeView.setHeaderHidden(True)
        treeModel = QStandardItemModel()
        rootNode = treeModel.invisibleRootItem()
        america = StandardItem("America", 16, setBold = True)
        california = StandardItem("California", 14)
        okland = StandardItem("Okland", 14)
        texas = StandardItem("Texas", 14)
        america.appendRow(california)
        canada = StandardItem("Canada", 16, setBold = True)
        california.appendRow(okland)
        rootNode.appendRow(america)
        rootNode.appendRow(canada)
        treeView.setModel(treeModel)
        treeView.expandAll()
        self.setCentralWidget(treeView)

app = QApplication(sys.argv)
demo = AppDemo()
demo.show()
sys.exit(app.exec_())


import os
import deepxde as dde
from matplotlib import pyplot as plt
import numpy as np
import skopt
from skopt import gp_minimize
from skopt.plots import plot_convergence, plot_objective
from skopt.space import Real, Categorical, Integer
from skopt.utils import use_named_args
import pandas as pd
from openpyxl import load_workbook
import time
import random
import imgkit
import torch
import wandb

if dde.backend.backend_name == "pytorch":
    sin = dde.backend.pytorch.sin
elif dde.backend.backend_name == "paddle":
    sin = dde.backend.paddle.sin
else:
    from deepxde.backend import tf

    sin = tf.sin

gp_seed = None
dde_seed = None
ITERATION = 0

current_file = os.path.abspath(__file__)
folder_pa = os.path.dirname(current_file)
# folder_path = f"{folder_pa}/"
folder_path = ""
output_path = None

epochs = 20000

def inizia_hpo():
    global output_path, gp_seed, dde_seed

    if gp_seed is None:
        gp_seed = random.randint(0, 1000)  # Genera il seed per GP
    if dde_seed is None:
        dde_seed = random.randint(0, 1000)  # Genera il seed per DDE

    output_path = f"{folder_path}hpo/{dde_seed}_{gp_seed}/"

    # Crea la struttura delle cartelle
    cartella_figure = f"{output_path}figures"
    cartella_history = f"{output_path}history"
    cartella_model = f"{output_path}model"

    # Crea le cartelle se non esistono già
    os.makedirs(cartella_figure, exist_ok=True)
    os.makedirs(cartella_history, exist_ok=True)
    os.makedirs(cartella_model, exist_ok=True)

    return output_path, gp_seed, dde_seed

def inizia_esperimento(ii, cc, ss):
    global output_path, dde_seed

    dde_seed = ii
    output_path = f"{folder_path}output/{ii}_{cc}_{ss}/"

    # Crea la struttura delle cartelle
    cartella_figure = f"{output_path}figures"
    cartella_history = f"{output_path}history"
    cartella_model = f"{output_path}model"

    # Crea le cartelle se non esistono già
    os.makedirs(cartella_figure, exist_ok=True)
    os.makedirs(cartella_history, exist_ok=True)
    os.makedirs(cartella_model, exist_ok=True)

    # Chiamata alla funzione per il codice specifico dell'esperimento
    # esegui_codice_specifico()

    # Restituisci i seed come output
    # return gp_seed, dde_seed, initial, ij
    return output_path, dde_seed

# General parameters
L0 = 0.05
TM = 45
Ta = 37
tauf = 1800

rho = 1050
c = 3639
k_eff = 5
alfa = rho * c / k_eff

W_avg = 2.3
W_min = 0.45
W_max = 4
cb = 3825

a1 = (alfa * (L0 ** 2)) / tauf
a2 = (L0 ** 2) * cb / k_eff

q0 = 16
dT = TM - Ta
q0_ad = q0/dT

# Network parameters
precision_train = 10
precision_test = 30

# HPO setting
n_calls = 100
dim_learning_rate = Real(low=1e-5, high=5e-1, name="learning_rate", prior="log-uniform")
dim_num_dense_layers = Integer(low=1, high=4, name="num_dense_layers")
dim_num_dense_nodes = Integer(low=5, high=500, name="num_dense_nodes")
dim_activation = Categorical(categories=["elu", "relu", "selu", "silu", "sigmoid", "sin", "swish", "tanh"],
                             name="activation")
dim_initialization = Categorical(categories=["Glorot normal", "Glorot uniform", "He normal", "He uniform"],
                             name="initialization")
# dim_weight_ic = Integer(low=1, high=1000000, name="weight_ic")
# dim_weight_bcl = Integer(low=1, high=1000000, name="weight_bcl")
# dim_weight_bcr = Integer(low=1, high=1000000, name="weight_bcr")
# dim_weight_domain = Integer(low=1, high=1000000, name="weight_domain")

dimensions = [
    dim_learning_rate,
    dim_num_dense_layers,
    dim_num_dense_nodes,
    dim_activation,
    dim_initialization
    # dim_weight_ic,
    # dim_weight_bcl,
    # dim_weight_bcr,
    # dim_weight_domain
]

# Prediction grid and ground truth
x = np.linspace(0, 1, 101)
t = np.linspace(0, 1, 101)
X, T = np.meshgrid(x, t)

def rescale(theta):
    return Ta + theta*(TM-Ta)

def cost(t):
    return np.full_like(t, q0_ad)

def lin(t):
    return q0_ad*(1-t)

def expo(t):
    return q0_ad*np.exp(-t)

def sinu(t):
    return q0_ad*np.cos(np.pi*t/2)

functions = {
    'c': cost,
    'l': lin,
    'e': expo,
    's': sinu
}

XX = {
    key: np.vstack((np.ravel(X), func(np.ravel(T)), np.ravel(T))).T
    for key, func in functions.items()
}

matlab = {'c': np.loadtxt(f"{folder_path}matlab/output_matlab_system_0.txt")[:, 2:],
          'l': np.loadtxt(f"{folder_path}matlab/output_matlab_system_1.txt")[:, 2:],
          'e': np.loadtxt(f"{folder_path}matlab/output_matlab_system_2.txt")[:, 2:],
          's': np.loadtxt(f"{folder_path}matlab/output_matlab_system_3.txt")[:, 2:]}

def pde(x, theta):
    dtheta_tau = dde.grad.jacobian(theta, x, i=0, j=2)
    dtheta_xx = dde.grad.hessian(theta, x, i=0, j=0)

    return a1 * dtheta_tau - dtheta_xx + a2 * theta * W_avg

def ic_func(x):
    return x[:, 1:2] * (x[:, 0:1] ** 4) / 4 + 15 * (((x[:, 0:1] - 1) ** 2) * x[:, 0:1]) / dT

def boundary_1(x, on_boundary):
    return on_boundary and np.isclose(x[0], 1)

def boundary_0(x, on_boundary):
    return on_boundary and np.isclose(x[0], 0)

def create_model(config, setts):
    global dde_seed
    dde.config.set_random_seed(dde_seed)

    learning_rate, num_dense_layers, num_dense_nodes, activation, initialization = config
    end_flux, end_time, w_domain, w_bcl, w_bcr, w_ic = setts

    geom = dde.geometry.Rectangle([0, 0], [1, end_flux])
    timedomain = dde.geometry.TimeDomain(0, end_time)
    geomtime = dde.geometry.GeometryXTime(geom, timedomain)

    bc_0 = dde.icbc.DirichletBC(geomtime, lambda x: 0, boundary_0)
    bc_1 = dde.icbc.NeumannBC(geomtime, lambda x: x[:, 1:2], boundary_1)

    ic = dde.icbc.IC(geomtime, ic_func, lambda _, on_initial: on_initial)

    data = dde.data.TimePDE(
        geomtime,
        pde,
        [bc_0, bc_1, ic],
        num_domain=2560,
        num_boundary=80,
        num_initial=160,
        num_test=2560,
    )

    net = dde.maps.FNN(
        [3] + [num_dense_nodes] * num_dense_layers + [1],
        activation,
        initialization,
    )

    loss_weights = [w_domain, w_bcl, w_bcr, w_ic]
    model = dde.Model(data, net)
    model.compile("adam", lr=learning_rate, loss_weights=loss_weights)
    return model

def train_model(model, config, setts):
    global output_path, dde_seed
    dde.config.set_random_seed(dde_seed)

    print(f"start training {config}_{setts}")
    losshistory, train_state = model.train(iterations=epochs,
                                           model_save_path=f"{output_path}model/{config}_{setts}.ckpt")
    dde.saveplot(losshistory, train_state, issave=True, isplot=False, loss_fname=f"{config}_{setts}_loss",
                 train_fname=f"{config}_{setts}_train", test_fname=f"{config}_{setts}_test",
                 output_dir=f"{output_path}history")

    train = np.array(losshistory.loss_train).sum(axis=1).ravel()
    test = np.array(losshistory.loss_test).sum(axis=1).ravel()
    metric = np.array(losshistory.metrics_test).sum(axis=1).ravel()

    # error = test.min()
    error = scarto(model)
    # return error
    return model

def scarto(model):
    pinns = {'c': model.predict(XX['c']), 'l': model.predict(XX['l']),
             'e': model.predict(XX['e']), 's': model.predict(XX['s'])}

    error = {'c': np.abs(matlab['c'] - pinns['c']), 'l': np.abs(matlab['l'] - pinns['l']),
             'e': np.abs(matlab['e'] - pinns['e']), 's': np.abs(matlab['s'] - pinns['s'])}

    error_values = [np.linalg.norm(error['c']), np.linalg.norm(error['l']),
                    np.linalg.norm(error['e']), np.linalg.norm(error['s'])]

    error_domain = [np.linalg.norm(np.abs(matlab['c'] - pinns['c'])),
                    np.linalg.norm(np.abs(matlab['l'] - pinns['l'])),
                    np.linalg.norm(np.abs(matlab['e'] - pinns['e'])),
                    np.linalg.norm(np.abs(matlab['s'] - pinns['s']))]

    # Reshape the error dictionary to have the same shape as the grid
    error_grid = {}
    for key in error.keys():
        error_grid[key] = np.reshape(error[key], X.shape)

    # Calculate the error evolution in time
    error_time = {
        'Constant Flux': np.linalg.norm(error_grid['c'], axis=1),
        'Linear Flux': np.linalg.norm(error_grid['l'], axis=1),
        'Exponential Flux': np.linalg.norm(error_grid['e'], axis=1),
        'Sinusoidal Flux': np.linalg.norm(error_grid['s'], axis=1)
    }

    # Calculate the error in space
    error_space = {
        'Constant Flux': np.linalg.norm(error_grid['c'], axis=0),
        'Linear Flux': np.linalg.norm(error_grid['l'], axis=0),
        'Exponential Flux': np.linalg.norm(error_grid['e'], axis=0),
        'Sinusoidal Flux': np.linalg.norm(error_grid['s'], axis=0)
    }

    # values = list(error_space.values())

    # return np.linalg.norm(values)
    return np.linalg.norm(error_domain)

def restore_model(model, config, setts):
    global output_path, dde_seed
    dde.config.set_random_seed(dde_seed)

    model.restore(f"{output_path}model/{config}_{setts}.ckpt-{epochs}.pt", verbose=0)
    return model
    # pinns = {'c': model.predict(XX['c']), 'l': model.predict(XX['l']),
    #          'e': model.predict(XX['e']), 's': model.predict(XX['s'])}
    #
    # error = [dde.metrics.l2_relative_error(matlab['c'], pinns['c']),
    #          dde.metrics.l2_relative_error(matlab['l'], pinns['l']),
    #          dde.metrics.l2_relative_error(matlab['e'], pinns['e']),
    #          dde.metrics.l2_relative_error(matlab['s'], pinns['s'])]
    #
    # e = np.linalg.norm(error)
    # return e

def restore2_model(model, config, setts):
    global output_path, dde_seed
    dde.config.set_random_seed(dde_seed)

    model.restore(f"{output_path}model/final.ckpt-{epochs}.pt", verbose=0)
    return model

def configure_subplot(ax, surface):
    ax.plot_surface(X, 1800 * T, surface, cmap='inferno', alpha=.8)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.w_yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.w_zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.tick_params(axis='both', labelsize=7, pad=2)
    ax.dist = 10
    ax.view_init(20, -120)

def plot_3d(p, confi, setts):
    global output_path, dde_seed
    dde.config.set_random_seed(dde_seed)

    pinns = {'c': p.predict(XX['c']), 'l': p.predict(XX['l']),
             'e': p.predict(XX['e']), 's': p.predict(XX['s'])}

    # Create 3D axes
    fig = plt.figure(figsize=(9, 12))

    # Define row and column titles
    row_titles = ['Constant Flux', 'Linear Flux', 'Exponential Flux', 'Sinusoidal Flux']
    col_titles = ['MATLAB', 'PINNs', 'Error']

    # Define surfaces for each subplot
    surfaces = [
        [matlab['c'].reshape(X.shape), pinns['c'].reshape(X.shape),
         np.abs(pinns['c'] - matlab['c']).reshape(X.shape)],
        [matlab['l'].reshape(X.shape), pinns['l'].reshape(X.shape),
         np.abs(pinns['l'] - matlab['l']).reshape(X.shape)],
        [matlab['e'].reshape(X.shape), pinns['e'].reshape(X.shape),
         np.abs(pinns['e'] - matlab['e']).reshape(X.shape)],
        [matlab['s'].reshape(X.shape), pinns['s'].reshape(X.shape),
         np.abs(pinns['s'] - matlab['s']).reshape(X.shape)]
    ]

    # Create a grid of subplots
    grid = plt.GridSpec(4, 3)

    # Iterate over rows and columns to add subplots
    for row in range(4):
        for col in range(3):
            ax = fig.add_subplot(grid[row, col], projection='3d')
            configure_subplot(ax, surfaces[row][col])

            # Set column titles for the top row
            if row == 0:
                ax.set_title(col_titles[col], fontsize=8, y=.96, weight='semibold')

            # Set row titles for the leftmost column
            if col == 0:
                ax.text2D(-0.2, 0.5, row_titles[row], fontsize=9, rotation='vertical', ha='center',
                          va='center', weight='semibold', transform=ax.transAxes)

    # Adjust spacing between subplots
    plt.subplots_adjust(wspace=0.15, hspace=0.25)

    # Save and show plot
    plt.savefig(f"{output_path}figures/plot3d_{confi}_{setts}.png", dpi=300, bbox_inches='tight')
    plt.show()

def plot2_3d(p, confi, setts):
    global output_path, dde_seed
    dde.config.set_random_seed(dde_seed)

    pinns = {'c': p.predict(XX['c']), 'l': p.predict(XX['l']),
             'e': p.predict(XX['e']), 's': p.predict(XX['s'])}

    # Create 3D axes
    fig = plt.figure(figsize=(9, 12))

    # Define row and column titles
    row_titles = ['Constant Flux', 'Linear Flux', 'Exponential Flux', 'Sinusoidal Flux']
    col_titles = ['MATLAB', 'PINNs', 'Error']

    # Define surfaces for each subplot
    surfaces = [
        [matlab['c'].reshape(X.shape), pinns['c'].reshape(X.shape),
         np.abs(pinns['c'] - matlab['c']).reshape(X.shape)],
        [matlab['l'].reshape(X.shape), pinns['l'].reshape(X.shape),
         np.abs(pinns['l'] - matlab['l']).reshape(X.shape)],
        [matlab['e'].reshape(X.shape), pinns['e'].reshape(X.shape),
         np.abs(pinns['e'] - matlab['e']).reshape(X.shape)],
        [matlab['s'].reshape(X.shape), pinns['s'].reshape(X.shape),
         np.abs(pinns['s'] - matlab['s']).reshape(X.shape)]
    ]

    # Create a grid of subplots
    grid = plt.GridSpec(4, 3)

    # Iterate over rows and columns to add subplots
    for row in range(4):
        for col in range(3):
            ax = fig.add_subplot(grid[row, col], projection='3d')
            configure_subplot(ax, surfaces[row][col])

            # Set column titles for the top row
            if row == 0:
                ax.set_title(col_titles[col], fontsize=8, y=.96, weight='semibold')

            # Set row titles for the leftmost column
            if col == 0:
                ax.text2D(-0.2, 0.5, row_titles[row], fontsize=9, rotation='vertical', ha='center',
                          va='center', weight='semibold', transform=ax.transAxes)

    # Adjust spacing between subplots
    plt.subplots_adjust(wspace=0.15, hspace=0.25)

    # Save and show plot
    plt.savefig(f"{output_path}figures/plot3d_final.png", dpi=300, bbox_inches='tight')
    plt.show()

@use_named_args(dimensions=dimensions)
def fitness(learning_rate, num_dense_layers, num_dense_nodes, activation, initialization):
    global ITERATION, gp_seed, dde_seed, output_path
    config = [learning_rate, num_dense_layers, num_dense_nodes, activation, initialization]
    dde.config.set_random_seed(dde_seed)

    # start a new wandb run to track this script
    wandb.init(
        # set the wandb project where this run will be logged
        project="hpo-no-source",

        # track hyperparameters and run metadata
        config={
            "learning rate": learning_rate,
            "num_dense_layers": num_dense_layers,
            "num_dense_nodes": num_dense_nodes,
            "activation": activation,
            "initialization": initialization
            # "weight_ic": weight_ic,
            # "weight_bcl": weight_bcl,
            # "weight_bcr": weight_bcr,
            # "weight_domain": weight_domain
        }
    )

    print(ITERATION, "it number")
    # Print the hyper-parameters.
    print("learning rate: {0:.1e}".format(learning_rate))
    print("num_dense_layers:", num_dense_layers)
    print("num_dense_nodes:", num_dense_nodes)
    print("activation:", activation)
    print("initialization:", initialization)
    # print("weight_ic:", weight_ic)
    # print("weight_bcl:", weight_bcl)
    # print("weight_bcr:", weight_bcr)
    # print("weight_domain:", weight_domain)
    print()

    start_time = time.time()

    # Create the neural network with these hyper-parameters.
    mm = create_model(config)
    # possibility to change where we save
    error = train_model(mm, config)
    # print(accuracy, 'accuracy is')

    if np.isnan(error):
        error = 10 ** 5

    end_time = time.time()
    time_spent = end_time - start_time

    # Store the configuration and error in a DataFrame
    data = {
        "Learning Rate": learning_rate,
        "Num Dense Layers": num_dense_layers,
        "Num Dense Nodes": num_dense_nodes,
        "Activation": activation,
        "Initialization": initialization,
        # "Weight_ic": weight_ic,
        # "Weight_bcl": weight_bcl,
        # "Weight_bcr": weight_bcr,
        # "Weight_domain": weight_domain,
        "Error": error,
        "Time Spent": time_spent
    }
    df = pd.DataFrame(data, index=[ITERATION])

    file_path = f"{output_path}hpo_results.csv"

    if not os.path.isfile(file_path):
        # Create a new CSV file with the header
        df.to_csv(file_path, index=False)
    else:
        # Append the DataFrame to the CSV file
        df.to_csv(file_path, mode='a', header=False, index=False)

    wandb.log({"err": error})
    wandb.finish()

    ITERATION += 1
    return error

def hpo(default_parameters):
    global gp_seed, dde_seed, output_path
    dde.config.set_random_seed(dde_seed)

    search_result = gp_minimize(
        func=fitness,
        dimensions=dimensions,
        acq_func="PI",  # Probability Improvement.
        n_calls=n_calls,
        x0=default_parameters,
        random_state=gp_seed,
    )

    print(search_result.x)

    # Plot convergence and save the figure
    plt.figure()
    plot_convergence(search_result)
    plt.title('Convergence Plot')
    plt.savefig(f"{output_path}figures/plot_conv.png",
            dpi=300, bbox_inches='tight')

    # Plot objective and save the figure
    plt.figure()
    plot_objective(search_result, show_points=True, size=3.8)
    plt.savefig(f"{output_path}figures/plot_obj.png",
                dpi=300, bbox_inches='tight')

    plt.show()

    # Load the CSV file into a pandas DataFrame
    csv_file = f'{output_path}hpo_results.csv'

    return search_result.x

def metric(row):
    L_1 = row['Num Dense Layers']
    N = row['Num Dense Nodes']
    N0 = 3
    net = np.ones(L_1+2) * N
    net[0] = N0
    net[L_1+1] = 1

    result = np.dot(net[:-1]+1, net[1:])
    return result

def reset_iteration():
    global ITERATION, gp_seed, dde_seed
    ITERATION = 0
    gp_seed = None
    dde_seed = None

def data_analysis(output_fold):
    # Navigate to the output folder
    os.chdir(output_fold)

    # Get the list of subdirectories in the output folder
    folders = [f for f in os.listdir(".") if os.path.isdir(f)]

    with pd.ExcelWriter("AA_RES.xlsx", engine="xlsxwriter") as writer:
        for folder in folders:
            # Create the full path to the folder
            folder_path = f"{output_fold}/{folder}"

            # Navigate to the current folder
            os.chdir(folder_path)

            # Read the hpo_results.csv file
            csv_file = "hpo_results.csv"
            a = pd.read_csv(csv_file)

            # Apply transformations to the dataframe a
            pd.options.display.float_format = '{:.2e}'.format
            a["Learning Rate"] = a["Learning Rate"].apply(lambda x: '%.2e' % x)
            a["Time Spent"] = a["Time Spent"].apply(lambda x: '%.2e' % x)
            a['Metric']=a.apply(metric, axis=1)
            a["Metric"] = a["Metric"].apply(lambda x: '%.2e' % x)
            a["Config"] = a[["Learning Rate", "Num Dense Layers", "Num Dense Nodes",
                             "Activation"]].apply(lambda x: '[' + ','.join(map(str, x)) + ']', axis=1)
            b = a.loc[:, ["Config", "Metric", "Error"]].sort_values(by="Error", ascending=True)
            b["Iteration"] = b.index

            # Navigate back to the output folder
            os.chdir(output_fold)
            b.to_excel(writer, sheet_name=folder, index=False)

def plot_err(b):
    global output_path, msg
    pinns = {'c': b.predict(XX['c']), 'l': b.predict(XX['l']),
             'e': b.predict(XX['e']), 's': b.predict(XX['s'])}

    error = {'x': np.ravel(X), 't': np.ravel(T),
             'c': np.abs(matlab['c'] - pinns['c']), 'l': np.abs(matlab['l'] - pinns['l']),
             'e': np.abs(matlab['e'] - pinns['e']), 's': np.abs(matlab['s'] - pinns['s'])}

    error_values = [np.linalg.norm(error['c']), np.linalg.norm(error['l']),
                    np.linalg.norm(error['e']), np.linalg.norm(error['s'])]

    x_labels = ['constant flux', 'linear flux', 'exponential flux', 'sinusoidal flux']

    # Create the bar graph
    plt.bar(x_labels, error_values)

    # Customize labels and title
    plt.xlabel('Error')
    plt.ylabel('Value')
    plt.title('Error Values')

    # Add labels to each bar
    # for i, value in enumerate(error_values):
    #     plt.text(i, value, str(round(value, 2)), ha='center', va='bottom')

    # Display the bar graph
    plt.savefig(f"{output_path}figures/error_domain_{msg}.png", dpi=300, bbox_inches='tight')
    plt.show()

    # Reshape the error dictionary to have the same shape as the grid
    error_grid = {}
    for key in error.keys():
        error_grid[key] = np.reshape(error[key], X.shape)

    # Calculate the error evolution in time
    error_evolution_time = {
        'Constant Flux': np.linalg.norm(error_grid['c'], axis=1),
        'Linear Flux': np.linalg.norm(error_grid['l'], axis=1),
        'Exponential Flux': np.linalg.norm(error_grid['e'], axis=1),
        'Sinusoidal Flux': np.linalg.norm(error_grid['s'], axis=1)
    }

    # Calculate the error in space
    error_in_space = {
        'Constant Flux': np.linalg.norm(error_grid['c'], axis=0),
        'Linear Flux': np.linalg.norm(error_grid['l'], axis=0),
        'Exponential Flux': np.linalg.norm(error_grid['e'], axis=0),
        'Sinusoidal Flux': np.linalg.norm(error_grid['s'], axis=0)
    }

    # Plot error evolution in time
    plt.figure(figsize=(10, 8))
    for i, (key, value) in enumerate(error_evolution_time.items()):
        plt.subplot(2, 2, i + 1)
        plt.plot(t, value)
        plt.title(key)
        plt.xlabel('Time')
        plt.ylabel('Error')
    plt.tight_layout()
    plt.savefig(f"{output_path}figures/error_time_{msg}.png", dpi=300, bbox_inches='tight')
    plt.show()

    # Plot error in space
    plt.figure(figsize=(10, 8))
    for i, (key, value) in enumerate(error_in_space.items()):
        plt.subplot(2, 2, i + 1)
        plt.plot(x, value)
        plt.title(key)
        plt.xlabel('Space')
        plt.ylabel('Error')
    plt.tight_layout()
    plt.savefig(f"{output_path}figures/error_space_{msg}.png", dpi=300, bbox_inches='tight')
    plt.show()

    # Save the data as an XLSX file
    data = {'Constant Flux': [np.linalg.norm(error_grid['c']), np.sum(np.linalg.norm(error_grid['c'], axis=0)), np.sum(np.linalg.norm(error_grid['c'], axis=1))],
            'Linear Flux': [np.linalg.norm(error_grid['l']), np.sum(np.linalg.norm(error_grid['l'], axis=0)), np.sum(np.linalg.norm(error_grid['l'], axis=1))],
            'Exponential Flux': [np.linalg.norm(error_grid['e']), np.sum(np.linalg.norm(error_grid['e'], axis=0)), np.sum(np.linalg.norm(error_grid['e'], axis=1))],
            'Sinusoidal Flux': [np.linalg.norm(error_grid['s']), np.sum(np.linalg.norm(error_grid['s'], axis=0)), np.sum(np.linalg.norm(error_grid['s'], axis=1))]}

    df = pd.DataFrame(data, index=['Domain', 'Space', 'Time'])
    df.to_excel(f"{output_path}/error.xlsx")


import os
import sys
from io import BytesIO, IOBase

def main():
    t = int(input())
    for _ in range(t):
        n = int(input())
        A = list(map(int, input().rstrip().split()))
        MaximumCandies = list(map(int, input().rstrip().split()))
        distA = [0]
        distB = [0]
        for pos in range(n):
            distA.append(distA[-1]+A[pos])
            distB.append(distB[-1]+MaximumCandies[pos])
        timeTogether = 0
        pos = 0
        while pos < n:
            if distA[pos] == distB[pos]:
                check = True
                while pos < n and A[pos] == MaximumCandies[pos] :
                    check = False
                    timeTogether += A[pos]
                    pos += 1
                if check:
                    pos+=1
                continue
            pos += 1
        print(timeTogether)

##############################################################################################################
BUFSIZE = 8192

class FastIO(IOBase):
    newlines = 0

    def __init__(self, file):
        self._fd = file.fileno()
        self.buffer = BytesIO()
        self.writable = "x" in file.mode or "r" not in file.mode
        self.write = self.buffer.write if self.writable else None

    def read(self):
        while True:
            b = os.read(self._fd, max(os.fstat(self._fd).st_size, BUFSIZE))
            if not b:
                break
            ptr = self.buffer.tell()
            self.buffer.seek(0, 2), self.buffer.write(b), self.buffer.seek(ptr)
        self.newlines = 0
        return self.buffer.read()

    def readline(self):
        while self.newlines == 0:
            b = os.read(self._fd, max(os.fstat(self._fd).st_size, BUFSIZE))
            self.newlines = b.count(b"\n") + (not b)
            ptr = self.buffer.tell()
            self.buffer.seek(0, 2), self.buffer.write(b), self.buffer.seek(ptr)
        self.newlines -= 1
        return self.buffer.readline()

    def flush(self):
        if self.writable:
            os.write(self._fd, self.buffer.getvalue())
            self.buffer.truncate(0), self.buffer.seek(0)

class IOWrapper(IOBase):
    def __init__(self, file):
        self.buffer = FastIO(file)
        self.flush = self.buffer.flush
        self.writable = self.buffer.writable
        self.write = lambda s: self.buffer.write(s.encode("ascii"))
        self.read = lambda: self.buffer.read().decode("ascii")
        self.readline = lambda: self.buffer.readline().decode("ascii")

sys.stdin, sys.stdout = IOWrapper(sys.stdin), IOWrapper(sys.stdout)
input = lambda: sys.stdin.readline().rstrip("\r\n")

if __name__ == "__main__":
    main()


import numpy as np
import scipy.special as sps

def the_transform(x,Nt = 500,Np = 500):

    N  = x.shape[0] - 1
    thetas = np.linspace(1e-10,np.pi,Nt)
    phis   = np.linspace(0,2*np.pi,Np)

    # duplicate the signal
    X = np.kron(x,np.ones((Np,Nt,1))).T

    # maps of angles
    Thetas, Phis = np.meshgrid(thetas,phis,indexing='ij')

    # map of phase space variable
    z = np.cos(Thetas/2)/np.sin(Thetas/2)*np.exp(1j*Phis)
    Z = np.kron(z,np.ones((N+1,1,1)))

    # indices
    ell = np.arange(N+1)
    L = np.kron(ell,np.ones((Np,Nt,1))).T

    # compute the transform using the generative function trick
    normZ  = np.sqrt(2*(1+np.abs(Z)**2))
    sumand = X*np.sqrt(sps.comb(N,L))*((1-Z)/(normZ))**L*((1+Z)/(normZ))**(N-L)
    Kz = np.sum(sumand,0)

    return Kz

def the_zeros(Kz, N, Nt = 500, Np = 500,feedback = False):

    thetas = np.linspace(1e-10,np.pi,Nt)
    phis   = np.linspace(0,2*np.pi,Np)

    # compute the zeros
    zx, zy    = extr2min(np.abs(Kz))

    # coordinates of the zeros on the sphre
    zp = phis[zy]
    zt = thetas[zx]

    # give some feedback
    if feedback:
        print('Local minima method has found '+str(zx.shape[0])+' zeros compared to the '+str(N)+' expected.')

    return zt, zp

def extr2min(M):
    central = M[1:-1, 1:-1]
    mask = np.full(central.shape, True, dtype=bool)
    sub_indices = (
        (np.s_[2:], np.s_[1:-1]),
        (np.s_[:-2], np.s_[1:-1]),
        (np.s_[1:-1], np.s_[2:]),
        (np.s_[1:-1], np.s_[:-2]),
        (np.s_[:-2], np.s_[:-2]),
        (np.s_[:-2], np.s_[2:]),
        (np.s_[2:], np.s_[2:]),
        (np.s_[2:], np.s_[:-2]),
    )
    for I, J in sub_indices:
        np.logical_and(mask, central <= M[I, J], out=mask, where=mask)

    x, y = np.where(mask)
    return x + 1, y + 1


from flask import Response
from flask_restful import Resource
from html5lib import serialize
from models import LikePost, db
import json
from . import can_view_post
from my_decorators import post_id_is_integer_or_400_error, id_is_integer_or_400_error, test
import flask_jwt_extended

class PostLikesListEndpoint(Resource):

    def __init__(self, current_user):
        self.current_user = current_user

    @flask_jwt_extended.jwt_required()
    @post_id_is_integer_or_400_error
    def post(self, post_id):
        if not can_view_post(post_id, self.current_user):
            return Response(json.dumps({'message': 'Like does not exist'}), mimetype="application/json", status=404)
        like = LikePost(self.current_user.id, post_id)
        db.session.add(like)
        db.session.commit()
        return Response(json.dumps(like.to_dict()), mimetype="application/json", status=201)

class PostLikesDetailEndpoint(Resource):

    def __init__(self, current_user):
        self.current_user = current_user

    @flask_jwt_extended.jwt_required()
    def delete(self, post_id, id):
        try:
            int(post_id)
            int(id)
        except:
            return Response(
                json.dumps({'message': '{0} must be an integer.'.format(id)}),
                mimetype="application/json",
                status=400
            )
        like = LikePost.query.get(id)
        if not like or like.user_id != self.current_user.id or not can_view_post(post_id, self.current_user):
            return Response(json.dumps({'message': 'Like does not exist'}), mimetype="application/json", status=404)

        LikePost.query.filter_by(id=id).delete()
        db.session.commit()
        serialized_data = {
            'message': 'Like {0} successfully deleted.'.format(id)
        }
        return Response(json.dumps(serialized_data), mimetype="application/json", status=200)

def initialize_routes(api):
    api.add_resource(
        PostLikesListEndpoint,
        '/api/posts/<post_id>/likes',
        '/api/posts/<post_id>/likes/',
        resource_class_kwargs={'current_user': flask_jwt_extended.current_user}
    )

    api.add_resource(
        PostLikesDetailEndpoint,
        '/api/posts/<post_id>/likes/<id>',
        '/api/posts/<post_id>/likes/<id>/',
        resource_class_kwargs={'current_user': flask_jwt_extended.current_user}
    )


from sql.select_tables.utils.data import DataManager
from nltk.util import ngrams
from typing import List
from tqdm import tqdm
from json import dump
from sql.select_tables.config import parse_args

def _ngram_process(s: str, n: List[int]) -> List[str]:
    ngram = []
    for x in n:
        ngram = ngram + list(ngrams(s, x))
    ngram = ["".join(x) for x in ngram]
    ngram = list(set(ngram))
    return ngram

def _check_same_gram(a: List[str], b: List[str]) -> int:
    a.sort()
    b.sort()

    i = 0
    j = 0
    cnt = 0
    while i < len(a) and j < len(b):
        if a[i] < b[j]:
            i += 1
        elif a[i] > b[j]:
            j += 1
        else:
            cnt += 1
            i += 1
            j += 1
    return cnt

def get_candidate_tables(predict_data_path, schema_path, content_path,
                         ngram=(1, 2, 3, 4),
                         answer_size=6,
                         return_eval=False,
                         max_length=None):
    dm: DataManager = DataManager.load(
        predict_data_path, schema_path, content_path, max_length)
    lam = [0.1, 0.1, 0.1, 5, 5, 5, 5, 5, 5, 5]
    used_rec_gram = [True, True, True, False, False, False, False, False, False, False]

    real_answer: List[List[int]] = []
    predict_answer: List[List[int]] = []
    for q in tqdm(dm.questions):
        current_question = q.question
        current_schema = q.schema
        current_real_answer = q.tables

        real_answer.append(current_real_answer)
        predict_answer.append([])
        candidate_tables = list(range(len(current_schema)))
        question_ngram = _ngram_process(current_question, ngram)
        rec_gram = []

        for i in range(answer_size):
            best_answer = -1
            best_score = -1
            best_gram = []
            for t in candidate_tables:
                if used_rec_gram[i]:
                    current_question_gram = list(
                        set(question_ngram) - set(rec_gram))
                else:
                    current_question_gram = question_ngram

                current_table = current_schema[t]
                table_name_ngram = _ngram_process(
                    current_table.split('[SEP]')[0], ngram)
                schema_ngram = _ngram_process(current_table, ngram)
                current_score = lam[i] * _check_same_gram(
                    current_question_gram, schema_ngram) + _check_same_gram(current_question_gram, table_name_ngram)
                if current_score > best_score:
                    best_score = current_score
                    best_answer = t
                    best_gram = table_name_ngram

            if i == 0:
                candidate_tables = []
            predict_answer[-1].append(best_answer)
            candidate_tables = candidate_tables + dm.foreign_key[best_answer]
            candidate_tables = list(
                set(candidate_tables) - set(predict_answer[-1]))
            rec_gram = list(set(rec_gram + best_gram))

    recall = 0
    example_recall = 0
    # f1 = 0
    result = []
    for i in range(len(predict_answer)):
        cnt = _check_same_gram(predict_answer[i], real_answer[i])
        current_recall = cnt / len(real_answer[i])
        example_recall += 1 if current_recall == 1 else 0
        recall += current_recall

        predict_table = [dm.table_name[x]
                         for x in predict_answer[i]]
        real_table = [dm.table_name[x]
                      for x in real_answer[i]]
        question = dm.questions[i].question

        # if cnt != len(real_answer[i]):
        result.append(
            {'question': question,
             'question_id': dm.questions[i].question_id,
             'real_tables': real_table,
             'predict_tables': predict_table,
             'predict_tables_id': predict_answer[i]})

    if return_eval:
        return result, {'recall (table level)': recall/len(predict_answer),
                        'recall (example level)': example_recall/len(predict_answer)}
    else:
        return result

def main():
    args = parse_args()
    result, eval = \
        get_candidate_tables(args.predict_data_path,
                             args.schema_path,
                             args.content_path,
                             args.ngram,
                             args.answer_size,
                             return_eval=True,
                             max_length=args.max_length)
    with open(args.result_path, 'w', encoding='utf-8') as f:
        dump(result, f, ensure_ascii=False, indent=4)

    print('\n'.join([f'{k}: {v}' for k,v in eval.items()]))

if __name__ == '__main__':
    main()