from .fuzzy_sets import FuzzySet, MF_object, Triangular_MF, SingletonsSet
from .rule_parsing import recursive_parse, preparse, postparse
from numpy import array, linspace, meshgrid, geomspace, log10, finfo, float64
from scipy.interpolate import interp1d
from copy import deepcopy
from collections import defaultdict, OrderedDict
import re
import string
from math import prod
try:
    from matplotlib.pyplot import figure, axes, plot, show, title, subplots, legend, colorbar
    matplotlib = True
except ImportError:
    matplotlib = False
try:
    import seaborn as sns
except:
    pass

# constant values
linestyles= ["-", "--", ":", "-."]

# for sanitization
valid_characters = string.ascii_letters + string.digits + "()_ "

class UndefinedUniverseOfDiscourseError(Exception):

    def __init__(self, message):
        self.message = message

class LinguisticVariable(object):
    """
        Creates a new linguistic variable.

        Args:
            FS_list: a list of FuzzySet instances.
            concept: a string providing a brief description of the concept represented by the linguistic variable (optional).
            universe_of_discourse: a list of two elements, specifying min and max of the universe of discourse. Optional, but it must be specified to exploit plotting facilities.
    """

    def __init__(self, FS_list=[], concept=None, universe_of_discourse=None):

        if FS_list==[]:
            raise Exception("ERROR: please specify at least one fuzzy set")
        self._universe_of_discourse = universe_of_discourse
        self._FSlist = FS_list
        self._concept = concept

    def get_values(self, v):
        result = {}
        for fs in self._FSlist:
            result[fs._term] = fs.get_value(v)
        return result

    def get_index(self, term):
        for n, fs in enumerate(self._FSlist):
            if fs._term == term: return n
        return -1

    def get_universe_of_discourse(self):
        """
        This method provides the leftmost and rightmost values of the universe of discourse of the linguistic variable.

        Returns:
            the two extreme values of the universe of discourse.
        """
        if self._universe_of_discourse is not None:
            return self._universe_of_discourse
        mins = []
        maxs = []
        try:
            for fs in self._FSlist:
                mins.append(min(fs._points.T[0]))
                maxs.append(max(fs._points.T[0]))
        except AttributeError:
            raise UndefinedUniverseOfDiscourseError("Cannot get the universe of discourse. Please, use point-based fuzzy sets or explicitly specify a universe of discourse")
        return min(mins), max(maxs)

    def draw(self, ax, TGT=None, element=None, highlight=None, xscale="linear"):
        """
        This method returns a matplotlib ax, representing all fuzzy sets contained in the liguistic variable.

        Args:
            ax: the matplotlib axis to plot to.
            TGT: (deprecated) show the memberships of a specific element of discourse TGT in the figure.
            element: show the memberships of a specific element of discourse in the figure.
            highlight: string, indicating the linguistic term/fuzzy set to highlight in the plot.
            xscale: default "linear", supported scales "log". Changes the scale of the xaxis.
        Returns:
            A matplotlib axis, representing all fuzzy sets contained in the liguistic variable.
        """
        if matplotlib == False:
            raise Exception("ERROR: please, install matplotlib for plotting facilities")
        if TGT is not None:
            print("Warning: argument 'TGT' is deprecated and will be discontinued in future relseases. Please use 'element'")
            element = TGT

        mi, ma = self.get_universe_of_discourse()
        if xscale == "linear":
            x = linspace(mi, ma, 10000)
        elif xscale == "log":
            if mi < 0 and ma > 0:
                x = geomspace(mi, -finfo(float64).eps, 5000) + geomspace(finfo(float64).eps, ma, 5000)
                # raise Exception("ERROR: cannot plot in log scale with negative universe of discourse")
            elif mi == 0:
                x = geomspace(finfo(float64).eps, ma, 10000)
            else:
                x = geomspace(mi, ma, 10000)
        else:
            raise Exception("ERROR: scale "+xscale+" not supported.")

        if highlight is None:
            linestyles= ["-", "--", ":", "-."]
        else:
            linestyles= ["-"]*4

        if element is not None:
            ax.plot([element, element], [0.0, 1], color="red", linestyle="--", linewidth=2.0, label="Value")

        for nn, fs in enumerate(self._FSlist):

            # singleton
            if isinstance(fs, SingletonsSet):
                xs = [pair[0] for pair in fs._funpointer._pairs]
                ys = [pair[1] for pair in fs._funpointer._pairs]
                ax.vlines(x=xs, ymin=0.0, ymax=ys, linestyles=linestyles[nn%4], color=next(ax._get_lines.prop_cycler)['color'], label=fs._term)

            elif fs._type == "function":
                y = [fs.get_value(xx) for xx in x]
                color = next(ax._get_lines.prop_cycler)['color']
                lw = 1

                if highlight==fs._term:
                    color="red"
                    lw =5
                elif highlight is not None:
                    color="lightgray"
                ax.plot(x,y, linestyles[nn%4], lw=lw, label=fs._term, color=color)

                # plot the membership degree on the fuzzy set
                if element is not None:
                    top = fs.get_value(element)
                    cut_y = [min(top, yy) for yy in y]
                    ax.fill([x[0]]+list(x)+[x[-1]],[0]+cut_y+[0] ,  color=color, alpha=0.5,)

            else:
                sns.regplot(x=fs._points.T[0], y=fs._points.T[1], marker="d", color="red", fit_reg=False, ax=ax)
                f = interp1d(fs._points.T[0], fs._points.T[1], bounds_error=False, fill_value=(fs.boundary_values[0], fs.boundary_values[1]))
                ax.plot(x, f(x), linestyles[nn%4], label=fs._term,)

        ax.set_xlabel(self._concept)
        ax.set_ylabel("Membership degree")
        ax.set_ylim(bottom=-0.05, top=1.05)
        if xscale == "log":
            ax.set_xscale("symlog", linthresh=10e-2)
            ax.set_xlim(x[0], x[-1])
        if highlight is None: ax.legend(loc="best")
        return ax

    def plot(self, outputfile="", TGT=None, element=None, highlight=None, xscale="linear"):
        """
        Shows a plot representing all fuzzy sets contained in the liguistic variable.

        Args:
            outputfile: path and filename where the plot must be saved.
            TGT: (deprecated) show the memberships of a specific element of discourse TGT in the figure.
            element: show the memberships of a specific element of discourse in the figure.
            highlight: string, indicating the linguistic term/fuzzy set to highlight in the plot.
            xscale: default "linear", supported scales "log". Changes the scale of the xaxis.
        """
        if matplotlib == False:
            raise Exception("ERROR: please, install matplotlib for plotting facilities")
        if TGT is not None:
            print("Warning: argument 'TGT' is deprecated and will be discontinued in future relseases. Please use 'element'")
            element = TGT

        fig, ax = subplots(1,1)
        self.draw(ax=ax, element=element, highlight=highlight, xscale=xscale)

        if outputfile != "":
            fig.savefig(outputfile)

        show()

    def __repr__(self):
        if self._concept is None:
            text = "N/A"
        else:
            text = self._concept
        return "<Linguistic variable '"+text+"', contains fuzzy sets %s, universe of discourse: %s>" % (str(self._FSlist), str(self._universe_of_discourse))

class AutoTriangle(LinguisticVariable):
    """
        Creates a new linguistic variable, whose universe of discourse is automatically divided in a given number of fuzzy sets.
        The sets are all symmetrical, normalized, and for each element of the universe their memberships sum up to 1.

        Args:
            n_sets: (integer) number of fuzzy sets in which the universe of discourse must be divided.
            terms: list of strings containing linguistic terms for the fuzzy sets (must be appropriate to the number of fuzzy sets).
            universe_of_discourse: a list of two elements, specifying min and max of the universe of discourse.
            verbose: True/False, toggles verbose mode.
    """

    def __init__(self, n_sets=3, terms=None, universe_of_discourse=[0,1], verbose=False):

        if n_sets<2:
            raise Exception("Cannot create linguistic variable with less than 2 fuzzy sets.")

        control_points = [x*1/(n_sets-1) for x in range(n_sets)]
        low = universe_of_discourse[0]
        high = universe_of_discourse[1]
        control_points = [low + (high-low)*x for x in control_points]

        if terms is None:
            terms = ['case %d' % (i+1) for i in range(n_sets)]

        FS_list = []

        FS_list.append(FuzzySet(function=Triangular_MF(low,low,control_points[1]), term=terms[0]))

        for n in range(1, n_sets-1):
            FS_list.append(
                FuzzySet(function=Triangular_MF(control_points[n-1], control_points[n], control_points[n+1]),
                    term=terms[n])
            )

        FS_list.append( FuzzySet(function=Triangular_MF(control_points[-2], high, high), term=terms[-1] ))

        super().__init__(FS_list, universe_of_discourse=universe_of_discourse)

        if verbose:
            for fs in FS_list:
                print(fs, fs.get_term())

class FuzzySystem(object):

    """
        Creates a new fuzzy system.

        Args:
            operators: a list of strings, specifying fuzzy operators to be used instead of defaults. Currently supported operators: 'AND_PRODUCT'.
            show_banner: True/False, toggles display of banner.
            sanitize_input: sanitize variables' names to eliminate non-accepted characters (under development).
            verbose: True/False, toggles verbose mode.
    """

    def __init__(self, operators=None, show_banner=True, sanitize_input=False, verbose=True):

        self._rules = []
        self._lvs = OrderedDict()
        self._variables = OrderedDict()
        self._crispvalues = OrderedDict()
        self._outputfunctions = OrderedDict()
        self._outputfuzzysets = OrderedDict()

        self._constants = []

        self._operators = operators

        self._detected_type = None

        self._sanitize_input = sanitize_input
        if sanitize_input and verbose:
            print (" * Warning: Simpful rules sanitization is enabled, please pay attention to possible collisions of symbols.")

        if show_banner: self._banner()

    def _banner(self):
        import pkg_resources
        vrs = pkg_resources.get_distribution('simpful').version
        print("  ____  __  _  _  ____  ____  _  _  __   ")
        print(" / ___)(  )( \\/ )(  _ \\(  __)/ )( \\(  ) v%s " % vrs)
        print(" \\___ \\ )( / \\/ \\ ) __/ ) _) ) \\/ (/ (_/\\ ")
        print(" (____/(__)\\_)(_/(__)  (__)  \\____/\\____/")
        print()
        print(" Created by Marco S. Nobile (m.s.nobile@tue.nl)")
        print(" and Simone Spolaor (simone.spolaor@unimib.it)")
        print()

    def get_fuzzy_sets(self, variable_name):
        """
            Returns the list of FuzzySet objects associated to one linguistic variable.

            Args:
                variable_name: name of the linguistic variable.

            Returns:
                a list containing FuzzySet objects.
        """
        try:
            return self._lvs[variable_name]._FSlist
        except ValueError:
            raise Exception("ERROR: linguistic variable %s does not exist" % variable_name)

    def get_fuzzy_set(self, variable_name, fs_name):
        """
            Returns a FuzzySet object associated to a linguistic variable.

            Args:
                variable_name: name of the linguistic variable.
                fs_name: linguistic term associated to the fuzzy set.

            Returns:
                a FuzzySet object.
        """
        try:
            LV =  self._lvs[variable_name]
        except ValueError:
            raise Exception("ERROR: linguistic variable %s does not exist" % (variable_name))

        for fs in LV._FSlist:
            if fs._term==fs_name:       return fs

        raise Exception("ERROR: fuzzy set %s of linguistic variable %s does not exist" % (fs_name, variable_name))

    def set_variable(self, name, value, verbose=False):
        """
        Sets the numerical value of a linguistic variable.

        Args:
            name: name of the linguistic variables to be set.
            value: numerical value to be set.
            verbose: True/False, toggles verbose mode.
        """
        if self._sanitize_input: name = self._sanitize(name)
        try:
            value = float(value)
            self._variables[name] = value
            if verbose: print(" * Variable %s set to %f" % (name, value))
        except ValueError:
            raise Exception("ERROR: specified value for "+name+" is not an integer or float: "+value)

    def set_constant(self, name, value, verbose=False):
        """
        Sets the numerical value of a linguistic variable to a constant value (i.e. ignore fuzzy inference).

        Args:
            name: name of the linguistic variables to be set to a constant value.
            value: numerical value to be set.
            verbose: True/False, toggles verbose mode.
        """
        if self._sanitize_input: name = self._sanitize(name)
        try:
            value = float(value)
            self._variables[name] = value
            self._constants.append(name)
            if verbose: print(" * Variable %s set to a constant value %f" % (name, value))
        except ValueError:
            raise Exception("ERROR: specified value for "+name+" is not an integer or float: "+value)

    def add_rules_from_file(self, path, verbose=False):
        """
        Imports new fuzzy rules by reading the strings from a text file.

        Args:
            path: path to the file containing the rules.
            verbose: True/False, toggles verbose mode.
        """
        if path[-3:].lower()!=".xls" and path[-4:].lower()!=".xlsx":
            with open(path) as fi:
                rules_strings = fi.readlines()
            self.add_rules(rules_strings, verbose=verbose)
        else:
            raise NotImplementedError("Excel support not available.")

    def _sanitize(self, rule):
        new_rule = "".join(ch for ch in rule if ch in valid_characters)
        return new_rule

    def add_rules(self, rules, verbose=False):
        """
        Adds new fuzzy rules to the fuzzy system.

        Args:
            rules: list of fuzzy rules to be added. Rules must be specified as strings, respecting Simpful's syntax.
            verbose: True/False, toggles verbose mode.
        """
        for rule in rules:

            # optional: remove invalid symbols
            if self._sanitize_input: rule = self._sanitize(rule)

            parsed_antecedent = recursive_parse(preparse(rule), verbose=verbose, operators=self._operators)
            parsed_consequent = postparse(rule, verbose=verbose)
            self._rules.append( [parsed_antecedent, parsed_consequent] )
            if verbose:
                print(" * Added rule IF", parsed_antecedent, "THEN", parsed_consequent)
                print()
        if verbose: print(" * %d rules successfully added" % len(rules))

    def get_rules(self):
        """
        Returns the rule base of the fuzzy system.

        Returns:
                a list containing the fuzzy rules as strings, in the same order they were added.
        """
        rule_base = []

        for r in self._rules:
            ant = str(r[0])
            ant = ant.replace("c.(", "(")
            ant = ant.replace("f.(", "(")
            ant = "IF "+ant

            cons = r[1]
            cons = " THEN ("+cons[0]+" IS "+cons[1]+")"

            rule = ant + cons
            rule_base.append(rule)

        return rule_base

    def replace_rule(self, i, new_rule, verbose=False):
        """
        Replaces the i-th rule in the FuzzySystem object. Rules are stored in the same order they were added.

        Args:
            i: index of the rule to be replaced in the fuzzy system.
            new_rule: fuzzy rule to be used for the replacement.
            verbose: True/False, toggles verbose mode.
        """

        if len(self._rules)<i or i<0:
            raise Exception("ERROR: rule number %d does not exist")

        if self._sanitize_input:
            new_rule = self._sanitize(new_rule)
        parsed_antecedent = recursive_parse(preparse(new_rule), verbose=verbose, operators=self._operators)
        parsed_consequent = postparse(new_rule, verbose=verbose)
        self._rules[i]=[parsed_antecedent, parsed_consequent]

        if verbose:
            print( " * Rule", i, "replaced with new rule: IF", parsed_antecedent, "THEN", parsed_consequent)
            print()

    def add_linguistic_variable(self, name, LV, verbose=False):
        """
        Adds a new linguistic variable to the fuzzy system.

        Args:
            name: string containing the name of the linguistic variable.
            LV: linguistic variable object to be added to the fuzzy system.
            verbose: True/False, toggles verbose mode.
        """
        if self._sanitize_input: name = self._sanitize(name)
        self._lvs[name]=deepcopy(LV)
        if self._lvs[name]._concept is None:
            self._lvs[name]._concept = name
        # self._lvs[name]=deepcopy(LV)
        if verbose: print(" * Linguistic variable '%s' successfully added" % name)

    def set_crisp_output_value(self, name, value, verbose=False):
        """
        Adds a new crisp output value to the fuzzy system.

        Args:
            name: string containing the identifying name of the crisp output value.
            value: numerical value of the crisp output value to be added to the fuzzy system.
            verbose: True/False, toggles verbose mode.
        """
        if self._sanitize_input: name = self._sanitize(name)
        self._crispvalues[name]=value
        if verbose: print(" * Crisp output value for '%s' set to %f" % (name, value))
        self._set_model_type("Sugeno")

    def set_output_function(self, name, function, verbose=False):
        """
        Adds a new output function to the fuzzy system.

        Args:
            name: string containing the identifying name of the output function.
            function: string containing the output function to be added to the fuzzy system.
                The function specified in the string must use the names of linguistic variables contained in the fuzzy system object.
            verbose: True/False, toggles verbose mode.
        """
        if self._sanitize_input: name = self._sanitize(name)
        self._outputfunctions[name]=function
        if verbose: print(" * Output function for '%s' set to '%s'" % (name, function))
        self._set_model_type("Sugeno")

    def _set_model_type(self, model_type):
        if self._detected_type == "inconsistent": return
        if self._detected_type is  None:
            self._detected_type = model_type
            print (" * Detected %s model type" % model_type )
        elif self._detected_type != model_type:
            print("WARNING: model type is unclear (simpful detected %s, but a %s output was specified)" % (self._detected_type, model_type))
            self._detected_type = 'inconsistent'

    def get_firing_strengths(self, input_values=None):
        """
            If "input_values" is not provided, it returns a list of the firing strengths of the the rules,
            given the current state of input variables.
            "input_values" is an optional argument, in the form of a dictionary containing a list of input states for each variable.
            In this second case, it returns a 2D list of firing strengths of the given states.

            Args:
                input_values: dictionary where the keys are names of linguistic variables and the values are a list of input values for that variable.

            Returns:
                a list containing rules' firing strengths, or a 2D list containing rules' firing strengths for each given input state.
        """
        if input_values is None:
            results = [float(antecedent[0].evaluate(self)) for antecedent in self._rules]
            return results
        else:
            firings = []
            for i in range(0, len(list(input_values.values())[0])):
                for var in input_values.keys():
                    self.set_variable(var, input_values[var][i])
                res = self.get_firing_strengths()
                firings.append(res)
            return firings

    def mediate(self, outputs, antecedent, results, ignore_errors=False, ignore_warnings=False, verbose=False):

        final_result = {}

        list_crisp_values = [x[0] for x in self._crispvalues.items()]
        list_output_funs  = [x[0] for x in self._outputfunctions.items()]

        for output in outputs:
            if verbose:
                print(" * Processing output for variable '%s'" %  output)
                print("   whose universe of discourse is:", self._lvs[output].get_universe_of_discourse())
                print("   contains the following fuzzy sets:", self._lvs[output]._FSlist )

            num = 0
            den = 0

            for (ant, res) in zip(antecedent, results):
                outname = res[0]
                outterm = res[1]
                weight = float(res[2])
                crisp = True

                if verbose:
                    print(" ** Rule composition:", ant, "->", res, ", output variable: '%s'" % outname, "with term: '%s'" % outterm)

                if outname==output:
                    if outterm not in list_crisp_values:
                        crisp = False
                        if outterm not in list_output_funs:
                            raise Exception("ERROR: one rule calculates an output named '"
                                + outterm
                                + "', but I cannot find it among the output terms.\n"
                                + " --- PROBLEMATIC RULE:\n"
                                + "IF " + str(ant) + " THEN " + str(res))
                    if crisp:
                        crispvalue = self._crispvalues[outterm]
                    elif isinstance(self._outputfunctions[outterm], MF_object):
                        raise Exception("ERROR in consequent of rule %s.\nSugeno reasoning does not support output fuzzy sets." % ("IF " + str(ant) + " THEN " + str(res)))
                    else:
                        string_to_evaluate = self._outputfunctions[outterm]
                        for k,v in self._variables.items():
                            # old version
                            # string_to_evaluate = string_to_evaluate.replace(k,str(v))

                            # match a variable name preceeded or followed by non-alphanumeric and _ characters
                            # substitute it with its numerical value
                            string_to_evaluate = re.sub(r"(?P<front>\W|^)"+k+r"(?P<end>\W|$)", r"\g<front>"+str(v)+r"\g<end>", string_to_evaluate)
                        crispvalue = eval(string_to_evaluate)

                    try:
                        value = ant.evaluate(self)
                    except RuntimeError:
                        raise Exception("ERROR: one rule could not be evaluated\n"
                        + " --- PROBLEMATIC RULE:\n"
                        + "IF " + str(ant) + " THEN " + str(res) + "\n")

                    temp = value*crispvalue*weight
                    num += temp
                    den += value

            try:
                if den == 0.0:
                    final_result[output] = 0.0
                    if not ignore_warnings:
                        print("WARNING: the sum of rules' firing for variable '%s' is equal to 0. The result of the Sugeno inference was set to 0." % output)
                else:
                    final_result[output] = num / den

            except ArithmeticError:
                if ignore_errors:
                    print("WARNING: cannot perform Sugeno inference for variable '%s'. The variable appears only as antecedent in the rules or an arithmetic error occurred." % output)
                else:
                    raise Exception("ERROR: cannot perform Sugeno inference for variable '%s'. The variable appears only as antecedent in the rules or an arithmetic error occurred." % output)

        return final_result

    def mediate_Mamdani(self,
        outputs,
        antecedent,
        results,
        ignore_errors=False,
        ignore_warnings=False,
        verbose=False,
        subdivisions=1000,
        aggregation_function=max):

        final_result = {}

        for output in outputs:
            if verbose:
                print(" * Processing output for variable '%s'" %  output)
                print("   whose universe of discourse is:", self._lvs[output].get_universe_of_discourse())
                print("   contains the following fuzzy sets:", self._lvs[output]._FSlist )
            cuts_list = defaultdict(list)

            x0, x1 = self._lvs[output].get_universe_of_discourse()

            for (ant, res) in zip(antecedent, results):
                outname = res[0]
                outterm = res[1]
                weight = float(res[2])

                if verbose:
                    print(" ** Rule composition:", ant, "->", res, ", output variable: '%s'" % outname, "with term: '%s'" % outterm)

                if outname==output:
                    try:
                        value = ant.evaluate(self)
                    except RuntimeError:
                        raise Exception("ERROR: one rule could not be evaluated\n"
                        + " --- PROBLEMATIC RULE:\n"
                        + "IF " + str(ant) + " THEN " + str(res) + "\n")

                    #degree of satisfaction of the rule is multiplied by rule weight
                    cuts_list[outterm].append(value*weight)

            values = []
            weightedvalues = []
            integration_points = linspace(x0, x1, subdivisions)

            convenience_dict = {}
            for k in cuts_list.keys():
                convenience_dict[k] = self._lvs[output].get_index(k)
            if verbose: print ( " * Indices:", convenience_dict)

            for u in integration_points:
                comp_values = []
                for k, v_list in cuts_list.items():
                    for v in v_list:
                        n = convenience_dict[k]
                        fs_term = self._lvs[output]._FSlist[n]
                        result = float(fs_term.get_value_cut(u, cut=v))
                        comp_values.append(result)
                #keep = max(comp_values)
                keep = aggregation_function(comp_values)
                values.append(keep)
                weightedvalues.append(keep*u)

            sumwv = sum(weightedvalues)
            sumv = sum(values)

            try:
                if sumv == 0.0:
                    CoG = 0
                    if not ignore_warnings:
                        print("WARNING: the sum of rules' firing for variable '%s' is equal to 0. The result of the Mamdani inference was set to 0." % output)
                else:
                    CoG = sumwv/sumv

            except ArithmeticError:
                if ignore_errors:
                    print("WARNING: cannot perform Mamdani inference for variable '%s'. The variable appears only as antecedent in the rules or an arithmetic error occurred." % output)
                else:
                    raise Exception("ERROR: cannot perform Mamdani inference for variable '%s'. The variable appears only as antecedent in the rules or an arithmetic error occurred." % output)

            if verbose: print (" * Weighted values: %.2f\tValues: %.2f\tCoG: %.2f"% (sumwv, sumv, CoG))
            final_result[output] = CoG

        return final_result

    def Sugeno_inference(self, terms=None, ignore_errors=False, ignore_warnings=False, verbose=False):
        """
        Performs Sugeno fuzzy inference.

        Args:
            terms: list of the names of the variables on which inference must be performed. If empty, all variables appearing in the consequent of a fuzzy rule are inferred.
            ignore_errors: True/False, toggles the raising of errors during the inference.
            ignore_warnings: True/False, toggles the raising of warnings during the inference.
            verbose: True/False, toggles verbose mode.

        Returns:
            a dictionary, containing as keys the variables' names and as values their numerical inferred values.
        """
        if self._sanitize_input and terms is not None:
            terms = [self._sanitize(term) for term in terms]

        # default: inference on ALL rules/terms
        if terms == None:
            temp = [rule[1][0] for rule in self._rules]
            terms = list(set(temp))
        else:
            # get rid of duplicates in terms to infer
            terms = list(set(terms))
            for t in terms:
                if t not in set([rule[1][0] for rule in self._rules]):
                    raise Exception("ERROR: Variable "+t+" does not appear in any consequent.")

        array_rules = array(self._rules, dtype='object')
        # cheking if the rule base is weighted, if not add dummy weights
        for n, cons in enumerate(array_rules.T[1]):
            if len(cons)<3:
                cons = cons + ("1.0",)
                array_rules.T[1][n] = cons

        if len(self._constants)==0:
            result = self.mediate(terms, array_rules.T[0], array_rules.T[1], ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose)
        else:
            #remove constant variables from list of variables to infer
            ncost_terms = [t for t in terms if t not in self._constants]
            result = self.mediate(ncost_terms, array_rules.T[0], array_rules.T[1], ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose)
            #add values of constant variables
            cost_terms = [t for t in terms if t in self._constants]
            for name in cost_terms:
                result[name] = self._variables[name]

        return result

    def Mamdani_inference(self, terms=None, subdivisions=1000, aggregation_function=max, ignore_errors=False, ignore_warnings=False, verbose=False):
        """
        Performs Mamdani fuzzy inference.

        Args:
            terms: list of the names of the variables on which inference must be performed.If empty, all variables appearing in the consequent of a fuzzy rule are inferred.
            subdivisions: the number of integration steps to be performed for calculating fuzzy set area (default: 1000).
            aggregation_function: pointer to function used to aggregate fuzzy sets during Mamdani inference, default is max. Use Python sum function, or simpful's probor function for sum and probabilistic OR, respectively.
            ignore_errors: True/False, toggles the raising of errors during the inference.
            ignore_warnings: True/False, toggles the raising of warnings during the inference.
            verbose: True/False, toggles verbose mode.

        Returns:
            a dictionary, containing as keys the variables' names and as values their numerical inferred values.
        """
        if self._sanitize_input and terms is not None:
            terms = [self._sanitize(term) for term in terms]

        # default: inference on ALL rules/terms
        if terms == None:
            temp = [rule[1][0] for rule in self._rules]
            terms= list(set(temp))
        else:
            # get rid of duplicates in terms to infer
            terms = list(set(terms))
            for t in terms:
                if t not in set([rule[1][0] for rule in self._rules]):
                    raise Exception("ERROR: Variable "+t+" does not appear in any consequent.")

        array_rules = array(self._rules, dtype=object)
        # cheking if the rule base is weighted, if not add dummy weights
        for n, cons in enumerate(array_rules.T[1]):
            if len(cons)<3:
                cons = cons + ("1.0",)
                array_rules.T[1][n] = cons

        if len(self._constants)==0:
            result = self.mediate_Mamdani(terms, array_rules.T[0], array_rules.T[1], ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose, subdivisions=subdivisions, aggregation_function=aggregation_function)
        else:
            #remove constant variables from list of variables to infer
            ncost_terms = [t for t in terms if t not in self._constants]
            result = self.mediate_Mamdani(ncost_terms, array_rules.T[0], array_rules.T[1], ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose, subdivisions=subdivisions, aggregation_function=aggregation_function)
            #add values of constant variables
            cost_terms = [t for t in terms if t in self._constants]
            for name in cost_terms:
                result[name] = self._variables[name]

        return result

    def probabilistic_inference(self, terms=None, ignore_errors=False, ignore_warnings=False, verbose=False):
        raise NotImplementedError()

    def inference(self, terms=None, subdivisions=1000, aggregation_function=max, ignore_errors=False, ignore_warnings=False, verbose=False):
        """
        Performs the fuzzy inference, trying to automatically choose the correct inference engine.

        Args:
            terms: list of the names of the variables on which inference must be performed. If empty, all variables appearing in the consequent of a fuzzy rule are inferred.
            subdivisions: the number of integration steps to be performed for calculating fuzzy set area (default: 1000).
            aggregation_function: pointer to function used to aggregate fuzzy sets during Mamdani inference, default is max. Use Python sum function, or simpful's probor function for sum and probabilistic OR, respectively.
            ignore_errors: True/False, toggles the raising of errors during the inference.
            ignore_warnings: True/False, toggles the raising of warnings during the inference.
            verbose: True/False, toggles verbose mode.

        Returns:
            a dictionary, containing as keys the variables' names and as values their numerical inferred values.
        """
        if self._detected_type == "Sugeno":
            return self.Sugeno_inference(terms=terms, ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose)
        elif self._detected_type == "probabilistic":
            return self.probabilistic_inference(terms=terms, ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose)
        elif self._detected_type is None: # default
            return self.Mamdani_inference(terms=terms, subdivisions=subdivisions, ignore_errors=ignore_errors, ignore_warnings=ignore_warnings, verbose=verbose, aggregation_function=aggregation_function)
        else:
            raise Exception("ERROR: simpful could not detect the model type, please use either Sugeno_inference() or Mamdani_inference() methods.")

    def plot_variable(self, var_name, outputfile="", TGT=None, element=None, highlight=None, ax=None, xscale="linear"):
        """
        Plots all fuzzy sets contained in a liguistic variable. Options for saving the figure and draw on a matplotlib ax are provided.

        Args:
            var_name: string containing the name of the linguistic variable to plot.
            outputfile: string containing path and filename where the plot must be saved.
            TGT: (deprecated) show the memberships of a specific element of discourse TGT in the figure.
            element: show the memberships of a specific element of discourse in the figure.
            highlight: string, indicating the linguistic term/fuzzy set to highlight in the plot.
            ax: a matplotlib ax where the variable will be plotted.
            xscale: default "linear", supported scales "log". Changes the scale of the xaxis.
        """
        if matplotlib == False:
            raise Exception("ERROR: please, install matplotlib for plotting facilities")
        if TGT is not None:
            print("Warning: argument 'TGT' is deprecated and will be discontinued in future relseases. Please use 'element'")
            element = TGT

        if ax != None:
            ax = self._lvs[var_name].draw(ax=ax, element=element, highlight=highlight, xscale=xscale)
            return ax
        self._lvs[var_name].plot(outputfile=outputfile, element=element, highlight=highlight, xscale=xscale)

    def produce_figure(self, outputfile="", max_figures_per_row=4, element_dict=None):
        """
        Plots the membership functions of each linguistic variable contained in the fuzzy system.

        Args:
            outputfile: string containing path and filename where the plot must be saved.
            max_figures_per_row: maximum number of figures per row in the plot.
            element_dict: dictionary of elements of the universe of discourse whose membership must be plotted over the fuzzy sets.
        """
        if matplotlib == False:
            raise Exception("ERROR: please, install matplotlib for plotting facilities")

        # use a defaultdict in case no target was specified
        if element_dict is None:
            element_dict = defaultdict(lambda: None)

        num_ling_variables = len(self._lvs)
        if num_ling_variables == 0:
            raise Exception("ERROR: no linguistic variables added. Please add at least one variable to plot")
        #print(" * Detected %d linguistic variables" % num_ling_variables)
        columns = min(num_ling_variables, max_figures_per_row)
        if num_ling_variables>max_figures_per_row:
            if num_ling_variables%max_figures_per_row==0:
                rows = num_ling_variables//max_figures_per_row
            else:
                rows = num_ling_variables//max_figures_per_row + 1
        else:
            rows = 1

        fig, ax = subplots(rows, columns, figsize=(columns*5, rows*5))

        if rows==1: ax = [ax]
        if columns==1: ax= [ax]

        n = 0
        for k, v in self._lvs.items():
            r = n%max_figures_per_row
            c = n//max_figures_per_row
            v.draw(ax[c][r], element=element_dict[k])
            ax[c][r].set_ylim(0,1.05)
            n+=1

        for m in range(n, columns*rows):
            r = m%max_figures_per_row
            c = m//max_figures_per_row
            ax[c][r].axis('off')

        fig.tight_layout()

        if outputfile != "":
            fig.savefig(outputfile)
        else:
            show()

    def plot_surface(self, variables, output, detail=40, color_map="plasma"):
        """
        Plots the surface induced by the rules.

        Args:
            variables: a pair of linguistic variables for the x and y axis.
            output: the output variable to be computed.
            detail: number of subdivisions along each axis.
            color_map: the color map to be used for the plot.

        Returns:
            a matplotlib figure object.
        """

        if len(variables)!=2:
            print("ERROR: please specify the two variables for the surface plot")
            return None

        v1, v2 = variables

        min_v1, max_v1 = self._lvs[v1].get_universe_of_discourse()
        min_v2, max_v2 = self._lvs[v2].get_universe_of_discourse()

        A = linspace(min_v1, max_v1, detail)
        B = linspace(min_v2, max_v2, detail)
        C = []

        for a in A:
            temp = []
            for b in B:
                self.set_variable(self._lvs[v1]._concept, a)
                self.set_variable(self._lvs[v2]._concept, b)
                res = self.Sugeno_inference()[output]
                temp.append( res )
            C.append(temp)
        C = array(C)

        A,B = meshgrid(A,B)

        fig = figure(figsize=(8,6))
        ax = axes(projection='3d')

        v = ax.plot_surface(A,B,C, shade=True, cmap=color_map)
        ax.set_xlabel(self._lvs[v1]._concept)
        ax.set_ylabel(self._lvs[v2]._concept)
        ax.set_zlabel(output)
        colorbar(v, ax=ax)
        fig.tight_layout()
        return fig

    def aggregate(self, list_variables, function):
        """
        Performs a fuzzy aggregation of linguistic variables contained in a FuzzySystem object.

        Args:
            list_variables: list of linguistic variables names in the FuzzySystem object to aggregate.
            function: pointer to an aggregation function. The function must accept as an argument a list of membership values.

        Returns:
            the aggregated membership values.
        """
        memberships = []
        for variable, fuzzyset in list_variables.items():
            value = self._variables[variable]
            result = self._lvs[variable].get_values(value)[fuzzyset]
            memberships.append(result)
        return function(memberships)

# useful pre-implemented functions for aggregation or implication during fuzzy inference
def prod(m_list):
    """
        Performs aggregation of membership values using the product operation.

        Args:
            m_list: list of membership values to aggregate.

        Returns:
            the aggregated membership value.
    """
    return prod(m_list)

def probor(m_list):
    """
        Performs aggregation of membership values using the probabilistic OR operation.

        Args:
            m_list: list of membership values to aggregate.

        Returns:
            the aggregated membership value.
    """
    res = m_list[0]
    if len(m_list) == 1:
        return m_list[0]
    for i in range(1, len(m_list)):
        res = res + m_list[i] - res * m_list[i]
    return res

if __name__ == '__main__':
    pass


import math
from collections import defaultdict

# 木を出力
def s_exps(sym_i_j):
    sym, i, j = sym_i_j.split(' ')
    if sym_i_j in best_edge:  # 非終端記号
        return f'({sym} {s_exps(best_edge[sym_i_j][0])} {s_exps(best_edge[sym_i_j][1])})'
    else:  # 終端記号
        return f'({sym} {words[int(i)]})'

# 文法を読み込む
# grammer_path = '../../test/08-grammar.txt'  # テスト用
grammer_path = '../../data/wiki-en-test.grammar'
nonterm = []  # （左、右１、右２、確率）の非終端記号
preterm = defaultdict(list)  # pre[右] = [（左、確率）...]形式のマップ
for line in open(grammer_path, 'r'):
    line = line.rstrip()
    lhs, rhs, prob = line.split('\t')
    rhs_symbols = rhs.split(' ')
    if len(rhs_symbols) == 1:  # 前終端記号
        preterm[rhs].append((lhs, math.log2(float(prob))))
    else:  # 非終端記号
        nonterm.append((lhs, rhs_symbols[0], rhs_symbols[1], math.log2(float(prob))))

# CKY
# input_path = '../../test/08-input.txt'  # テスト用
input_path = '../../data/wiki-en-short.tok'
for line in open(input_path, 'r'):
    words = line.rstrip().split(' ')
    best_score = defaultdict(lambda: -math.inf)  # sym_ij : 最大対数確率
    best_edge = {}  # sym_ij : (lsym_ik, rsym_kj)

    # 前終端記号を追加
    for i in range(len(words)):
        for lhs, log_prob in preterm[words[i]]:
            best_score[f'{lhs} {i} {i+1}'] = log_prob

    # 非終端記号の組み合わせ
    for j in range(2, len(words)+1):  # jはスパンの右側
        for i in range(j-2, -1, -1):  # iはスパンの左側（右から左へ処理）
            for k in range(i+1, j):  # kはrsymの開始点
                # 各文法ルールを展開: log(P(sym → lsym rsym)) = logprob
                for sym, lsym, rsym, logprob in nonterm:
                    # 両方の子供の確率が0より大きい
                    if best_score[f'{lsym} {i} {k}'] > -math.inf and \
                       best_score[f'{rsym} {k} {j}'] > -math.inf:
                        # このノード・辺の対数確率を計算
                        my_lp = best_score[f'{lsym} {i} {k}'] + \
                                best_score[f'{rsym} {k} {j}'] + logprob
                        # この辺が確率最大のものなら更新
                        if my_lp > best_score[f'{sym} {i} {j}']:
                            best_score[f'{sym} {i} {j}'] = my_lp
                            best_edge[f'{sym} {i} {j}'] = (f'{lsym} {i} {k}', f'{rsym} {k} {j}')

    # 構文木の出力
    print(s_exps(f'S 0 {len(words)}'))


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# file list
fileList = {
    "i_70_72": "CRD_scripts/files_for_python/70_72.ALLchr.module.merged.intersect.bed",
    "i_70_73": "CRD_scripts/files_for_python/70_73.ALLchr.module.merged.intersect.bed",
    "i_72_73": "CRD_scripts/files_for_python/72_73.ALLchr.module.merged.intersect.bed",
    "i_70_72_73": "CRD_scripts/files_for_python/70_72_73.ALLchr.module.merged.intersect.bed",

    "m_70": "CRD_scripts/files_for_python/EGAD00001002670.ALLchr.module.merged.bed",
    "m_72": "CRD_scripts/files_for_python/EGAD00001002672.ALLchr.module.merged.bed",
    "m_73": "CRD_scripts/files_for_python/EGAD00001002673.ALLchr.module.merged.bed"
}
# get nbr of bp per file and append in total_ bp
total_bp = pd.DataFrame(columns=['file', 'total_bp'])
T = {}
for file, filename in fileList.items():
    print(file)
    data = pd.read_csv(filename, sep="\t", header=None)
    data["bp"] = data[2] - data[1]
    total_bp_file = data["bp"].sum(axis=0)
    total_bp = total_bp.append({'file': file, 'total_bp': total_bp_file}, ignore_index=True)
    T[file] = total_bp_file

# matrix of overlap

overlap_array = ([[[1, T['i_70_72'] / T['m_70'], T['i_70_73'] / T['m_70']],
                   [T['i_70_72'] / T['m_72'], 1, T['i_72_73'] / T['m_72']],
                   [T['i_70_73'] / T['m_73'], T['i_72_73'] / T['m_73'], 1]]])

overlap_array = np.around(overlap_array, decimals=2)

# sns.heatmap(overlap_array)
# add empty diag and names and axes and title and round and size of rounds)

sns.set()
uniform_data = np.random.rand(3,3)
#ax = sns.heatmap(uniform_data, vmin=0, vmax=1,xticklabels=['NEUT','MONO','TCELL'], yticklabels=['NEUT','MONO','T-CELL'] )

ax = sns.heatmap(uniform_data, rasterized=False, annot=True, cmap="Blues", linewidths=4)
ax.set(xlabel='Query', ylabel='Reference', title='CRD Tissue Sharing')
plt.show()


import pygame
from os import path

img_dir = path.join(path.dirname(__file__), 'img5')
snd_dir = path.join(path.dirname(__file__), 'snd4')

pygame.display.set_caption('Выше-ниже')

FPS = 60
WIDTH = 900
HEIGHT = 575

WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
GRAY = (125, 125, 125)
LIGHT_BLUE = (64, 128, 255)
GREEN = (0, 200, 64)
YELLOW = (225, 225, 0)
PINK = (230, 50, 230)

pygame.init()
screen = pygame.display.set_mode((WIDTH, HEIGHT))
clock = pygame.time.Clock()

chemodan_image = pygame.image.load(path.join(img_dir, 'чемодан.png'))
chemodan = chemodan_image.get_rect()
chemodan.topleft = (113, 370)

shkaf_image = pygame.image.load(path.join(img_dir, 'шкаф.png'))
shkaf = shkaf_image.get_rect()
shkaf.topleft = (580, 200)

music_image = pygame.image.load(path.join(img_dir, "музыка2.png"))
music_button = music_image.get_rect()
music_button.topleft = (0, 525)
music_image.set_colorkey(BLACK)

background = pygame.image.load(path.join(img_dir, "1.png")).convert()
background_rect = background.get_rect()

pygame.mixer.music.load(path.join(snd_dir, 'Autumn Forest.mp3'))
pygame.mixer.music.set_volume(0.4)

pygame.mixer.music.play()

vishe_nisze_sound = pygame.mixer.Sound(path.join(snd_dir,'Что выше.mp3'))
vishe_nisze_sound.play()

question_image = pygame.image.load(path.join(img_dir,'вопрос2.png')).convert()
question_button = question_image.get_rect()
question_button.topleft = (850,525)
question_image.set_colorkey(BLACK)

incorrect_sound = pygame.mixer.Sound(path.join(snd_dir, 'звук неправильного ответа.mp3'))
incorrect_sound.set_volume(0.35)

running = True
flPause = False
while running:
    clock.tick(FPS)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

        if event.type == pygame.MOUSEBUTTONDOWN:
            pos = pygame.mouse.get_pos()

            if shkaf.collidepoint(pos):
                print("Получилось")
                pygame.mixer.music.stop()
                vishe_nisze_sound.stop()
                sc = pygame.display.set_mode((500, 300))
                clock = pygame.time.Clock()
                start_time = pygame.time.get_ticks()
                f = pygame.font.Font(None, 100)

                molodec_image = pygame.image.load(path.join(img_dir, "Смайлик2.png"))
                molodec = molodec_image.get_rect()
                molodec.topleft = (150, 50)
                molodec_sound = pygame.mixer.Sound(path.join(snd_dir, 'Молодец.mp3'))
                molodec_sound.play()

                sc.blit(molodec_image, molodec)
                r = True
                while r == True:
                    sc.fill((255, 255, 255))
                    for i in pygame.event.get():
                        if i.type == pygame.QUIT:
                            exit()

                    clock.tick(FPS)

                    if start_time and pygame.time.get_ticks() - start_time > 2000:
                        r = False
                    sc.blit(molodec_image, molodec)
                    pygame.display.update()
                import Diplom.shire_uzhe
            else:
                incorrect_sound.play()

            if music_button.collidepoint(pos):
                incorrect_sound.stop()
                flPause = not flPause
                if flPause:
                    pygame.mixer.music.pause()
                else:
                    pygame.mixer.music.unpause()

            if question_button.collidepoint(pos):
                vishe_nisze_sound.stop()
                incorrect_sound.stop()
                vishe_nisze_sound.play()

    screen.blit(background, background_rect)

    screen.blit(chemodan_image, chemodan)
    screen.blit(shkaf_image, shkaf)
    screen.blit(music_image, music_button)
    screen.blit(question_image, question_button)

    pygame.display.flip()

pygame.quit()


"""Test Force."""
from asyncio import sleep
from typing import List, Union, Tuple, Optional

from opentrons.hardware_control.ot3api import OT3API
from opentrons.hardware_control.types import GripperJawState

from hardware_testing.drivers import find_port, list_ports_and_select
from hardware_testing.drivers.mark10 import Mark10, SimMark10
from hardware_testing.data.csv_report import (
    CSVReport,
    CSVLine,
    CSVResult,
    CSVLineRepeating,
)

from hardware_testing.data import ui
from hardware_testing.opentrons_api import helpers_ot3
from hardware_testing.opentrons_api.types import Axis, OT3Mount, Point

SLOT_FORCE_GAUGE = 4

GRIP_DUTY_CYCLES: List[int] = [40, 30, 25, 20, 15, 10, 6]
NUM_DUTY_CYCLE_TRIALS = 20

GRIP_FORCES_NEWTON: List[int] = [20, 15, 10, 5]
NUM_NEWTONS_TRIALS = 1
FAILURE_THRESHOLD_PERCENTAGES = [20, 20, 20, 40]

WARMUP_SECONDS = 10

FORCE_GAUGE_TRIAL_SAMPLE_INTERVAL = 0.25  # seconds
FORCE_GAUGE_TRIAL_SAMPLE_COUNT = 20  # 20 samples = 5 seconds @ 4Hz

GAUGE_OFFSET = Point(x=2, y=-42, z=75)

def _get_test_tag(
    trial: int,
    newtons: Optional[int] = None,
    duty_cycle: Optional[int] = None,
) -> str:
    if newtons and duty_cycle:
        raise ValueError("must measure either force or duty-cycle, not both")
    if newtons is None and duty_cycle is None:
        raise ValueError("both newtons and duty-cycle are None")
    if newtons is not None:
        return f"newtons-{newtons}-trial-{trial + 1}"
    else:
        return f"duty-cycle-{duty_cycle}-trial-{trial + 1}"

def _get_gauge(is_simulating: bool) -> Union[Mark10, SimMark10]:
    if is_simulating:
        return SimMark10()
    else:
        try:
            port = find_port(*Mark10.vid_pid())
        except RuntimeError:
            port = list_ports_and_select("Mark10 Force Gauge")
        print(f"Setting up force gauge at port: {port}")
        return Mark10.create(port=port)

def _get_force_gauge_hover_and_grip_positions(api: OT3API) -> Tuple[Point, Point]:
    grip_pos = helpers_ot3.get_slot_calibration_square_position_ot3(SLOT_FORCE_GAUGE)
    grip_pos += GAUGE_OFFSET
    hover_pos = grip_pos._replace(z=api.get_instrument_max_height(OT3Mount.GRIPPER))
    return hover_pos, grip_pos

def build_csv_lines() -> List[Union[CSVLine, CSVLineRepeating]]:
    """Build CSV Lines."""
    lines: List[Union[CSVLine, CSVLineRepeating]] = list()
    for force in GRIP_FORCES_NEWTON:
        for trial in range(NUM_NEWTONS_TRIALS):
            tag = _get_test_tag(trial, newtons=force)
            force_data_types = [float] * FORCE_GAUGE_TRIAL_SAMPLE_COUNT
            lines.append(CSVLine(f"{tag}-data", force_data_types))
            lines.append(CSVLine(f"{tag}-average", [float]))
            lines.append(CSVLine(f"{tag}-target", [float]))
            lines.append(CSVLine(f"{tag}-pass-%", [float]))
            lines.append(CSVLine(f"{tag}-result", [CSVResult]))
    for duty_cycle in GRIP_DUTY_CYCLES:
        for trial in range(NUM_DUTY_CYCLE_TRIALS):
            tag = _get_test_tag(trial, duty_cycle=duty_cycle)
            force_data_types = [float] * FORCE_GAUGE_TRIAL_SAMPLE_COUNT
            lines.append(CSVLine(f"{tag}-data", force_data_types))
            lines.append(CSVLine(f"{tag}-average", [float]))
            lines.append(CSVLine(f"{tag}-duty-cycle", [float]))
    return lines

async def _read_forces(gauge: Union[Mark10, SimMark10]) -> List[float]:
    n = list()
    for _ in range(FORCE_GAUGE_TRIAL_SAMPLE_COUNT):
        force = gauge.read_force()
        n.append(force)
        if not gauge.is_simulator():
            await sleep(FORCE_GAUGE_TRIAL_SAMPLE_INTERVAL)
    return n

async def _grip_and_read_forces(
    api: OT3API,
    gauge: Union[Mark10, SimMark10],
    force: Optional[int] = None,
    duty: Optional[int] = None,
) -> List[float]:
    if not api.is_simulator:
        await sleep(2)  # let sensor settle
    if duty is not None:
        await api._grip(duty_cycle=float(duty))
        api._gripper_handler.set_jaw_state(GripperJawState.GRIPPING)
    else:
        assert force is not None
        await api.grip(float(force))
    if gauge.is_simulator():
        if duty is not None:
            gauge.set_simulation_force(float(duty) * 0.5)  # type: ignore[union-attr]
        elif force is not None:
            gauge.set_simulation_force(float(force))  # type: ignore[union-attr]
    ret_list = await _read_forces(gauge)
    await api.ungrip()
    return ret_list

async def _setup(api: OT3API) -> Union[Mark10, SimMark10]:
    z_ax = Axis.Z_G
    g_ax = Axis.G
    mount = OT3Mount.GRIPPER

    # OPERATOR SETS UP GAUGE
    ui.print_header("SETUP FORCE GAUGE")
    if not api.is_simulator:
        ui.get_user_ready(f"add gauge to slot {SLOT_FORCE_GAUGE}")
        ui.get_user_ready("plug gauge into USB port on OT3")
    gauge = _get_gauge(api.is_simulator)
    gauge.connect()
    print("test readings")
    ret_list = await _read_forces(gauge)
    print(ret_list)

    # HOME
    print("homing Z and G...")
    await api.home([z_ax, g_ax])
    # MOVE TO GAUGE
    await api.ungrip()
    _, target_pos = _get_force_gauge_hover_and_grip_positions(api)
    await helpers_ot3.move_to_arched_ot3(api, mount, target_pos + Point(z=15))
    if not api.is_simulator:
        ui.get_user_ready("please make sure the gauge in the middle of the gripper")
    await helpers_ot3.jog_mount_ot3(api, OT3Mount.GRIPPER)
    if not api.is_simulator:
        ui.get_user_ready("about to grip")
    await api.grip(20)
    for sec in range(WARMUP_SECONDS):
        print(f"warmup ({sec + 1}/{WARMUP_SECONDS})")
        if not api.is_simulator:
            await sleep(1)
    await api.ungrip()
    return gauge

async def run_increment(api: OT3API, report: CSVReport, section: str) -> None:
    """Run Increment."""
    gauge = await _setup(api)

    # LOOP THROUGH DUTY-CYCLES
    ui.print_header("MEASURE DUTY-CYCLES")
    for duty_cycle in GRIP_DUTY_CYCLES:
        # GRIP AND MEASURE FORCE
        for trial in range(NUM_DUTY_CYCLE_TRIALS):
            print(
                f"{duty_cycle}% duty cycle - trial {trial + 1}/{NUM_DUTY_CYCLE_TRIALS}"
            )
            actual_forces = await _grip_and_read_forces(api, gauge, duty=duty_cycle)
            print(actual_forces)
            avg_force = sum(actual_forces) / len(actual_forces)
            print(f"average = {round(avg_force, 2)} N")
            tag = _get_test_tag(trial, duty_cycle=duty_cycle)
            report(section, f"{tag}-data", actual_forces)
            report(section, f"{tag}-average", [avg_force])
            report(section, f"{tag}-duty-cycle", [duty_cycle])

    print("done")
    await api.retract(OT3Mount.GRIPPER)

async def run(api: OT3API, report: CSVReport, section: str) -> None:
    """Run."""
    gauge = await _setup(api)

    # LOOP THROUGH FORCES
    ui.print_header("MEASURE NEWTONS")
    for expected_force, allowed_percent_error in zip(
        GRIP_FORCES_NEWTON, FAILURE_THRESHOLD_PERCENTAGES
    ):
        for trial in range(NUM_NEWTONS_TRIALS):
            print(f"{expected_force}N - trial {trial + 1}/{NUM_NEWTONS_TRIALS}")
            actual_forces = await _grip_and_read_forces(
                api, gauge, force=expected_force
            )
            print(actual_forces)
            # base PASS/FAIL on average
            avg_force = sum(actual_forces) / len(actual_forces)
            print(f"average = {round(avg_force, 2)} N")
            error = (avg_force - expected_force) / expected_force
            result = CSVResult.from_bool(abs(error) * 100 < allowed_percent_error)
            # store all data in CSV
            tag = _get_test_tag(trial, newtons=expected_force)
            report(section, f"{tag}-data", actual_forces)
            report(section, f"{tag}-average", [avg_force])
            report(section, f"{tag}-target", [expected_force])
            report(section, f"{tag}-pass-%", [allowed_percent_error])
            report(section, f"{tag}-result", [result])

    print("done")
    await api.retract(OT3Mount.GRIPPER)


import datetime
import os

File = os.path.basename(__file__)

EPS = 1e-8

def make_orders(order_books, pair, current_balance, need_base, need_quote):
    """
    makes orders to rebalance assets
    :param order_books: see example at the beginning of this file
    :param pair: pair or currencies to balance assets for
    :param need_base: desired amount base currency
    :param need_quote: desired amount of quote currency
    :return: our_orders
    """
    our_orders = dict()
    # current_balance = copy.deepcopy(current_balance)

    currencies = pair.split('_')
    base_cur = currencies[0]  # base currency of current pair (BTC for BTC/USD)
    quote_cur = currencies[1]  # quote currency of current pair (USD for BTC/USD)
    # print(pair, base_cur, quote_cur)

    ax = 0
    bx = 0
    asks = order_books[pair]['asks']
    bids = order_books[pair]['bids']
    ask_count = len(asks)
    bid_count = len(bids)

    base_amount = 0  # required amount of base currency
    quote_amount = 0  # required amount of quote currency

    sell_orders = {}  # all sell_orders for current pair
    buy_orders = {}  # all buy_orders for current pair

    if need_base != 0.0: # BUY
        while ax < ask_count and need_base > EPS:
            ask_price = asks[ax][0]
            if ask_price < EPS:
                ax += 1
                continue

            ask_vol = asks[ax][1]
            if ask_vol < EPS:
                ax += 1
                continue

            ask_price_real = asks[ax][2]

            ask_exch = asks[ax][3]  # BID: base -> quote

            ask_bal = current_balance[ask_exch][quote_cur]
            if ask_bal < EPS:
                ax += 1
                continue

            try:
                m = min(ask_vol, ask_bal / ask_price_real, need_base)  # current micro-trade volume
                if m < EPS:
                    ax += 1
                    continue
                need_base -= m
                asks[ax][1] -= m
                current_balance[ask_exch][quote_cur] -= m * ask_price_real
            except ZeroDivisionError as e:
                Time = datetime.datetime.utcnow()
                EventType = "ZeroDivisionError"
                Function = "get_arb_opp"
                Explanation = "ask_price_real is equal to 0"
                EventText = e
                ExceptionType = type(e)
                print("{}|{}|{}|{}|{}|{}|{}".format(Time, EventType, Function, File, Explanation, EventText,
                                                    ExceptionType))
                break

            if ask_exch in buy_orders:
                buy_orders[ask_exch][0] = max(buy_orders[ask_exch][0], ask_price_real)
                buy_orders[ask_exch][1] += m
            else:
                buy_orders[ask_exch] = [ask_price_real, m]

    else: #SELL
        m = 0
        while bx < bid_count and need_quote > 1e-4:

            bid_price = bids[bx][0]
            if bid_price < EPS:
                bx += 1
                continue

            bid_vol = bids[bx][1]
            if bid_vol < EPS:
                bx += 1
                continue

            bid_price_real = bids[bx][2]

            bid_exch = bids[bx][3]  # ASK: quote -> base

            bid_bal = current_balance[bid_exch][base_cur]
            if bid_bal < EPS:
                bx += 1
                continue

            m = min(bid_vol, bid_bal, need_quote / bid_price_real)  # current micro-trade volume
            if m < EPS:
                bx += 1
                continue
            need_quote -= bid_price * m
            bids[bx][1] -= m
            current_balance[bid_exch][base_cur] -= m

            if bid_exch in sell_orders:
                sell_orders[bid_exch][0] = min(sell_orders[bid_exch][0], bid_price_real)
                sell_orders[bid_exch][1] += m
            else:
                sell_orders[bid_exch] = [bid_price_real, m]

    our_orders = {}
    our_orders['buy'] = buy_orders
    our_orders['sell'] = sell_orders
    our_orders['required_base_amount'] = base_amount
    our_orders['required_quote_amount'] = quote_amount
    our_orders['profit'] = 0

    return our_orders

def rebalance(data, order_books, total_balances, balances):
    ok = False
    orders = {}
    cur_pair = ''
    try:
        for pair in data:
            cnt = 0
            rate = 0
            for exch in data[pair]:
                if len(data[pair][exch]['bids']) != 0 and len(data[pair][exch]['asks']) != 0:
                    rate += float(data[pair][exch]['bids'][0][2])
                    rate += float(data[pair][exch]['asks'][0][2])
                    cnt += 2
            if cnt == 0:
                continue
            rate /= cnt
            base, quote = pair.split('_')
            if total_balances[base] == 0.0 and total_balances[quote] == 0.0:
                continue
            minvol, maxvol = min(total_balances[base] * rate, total_balances[quote]), max(total_balances[base] * rate, total_balances[quote])
            if minvol / maxvol < 0.8:
                ok = True
                cur_pair = pair
                if total_balances[base] * rate < total_balances[quote]:
                    orders = make_orders(order_books, pair, balances, need_base=(total_balances[quote] - total_balances[base] * rate) / 2 / rate, need_quote=0.0)
                else:
                    orders = make_orders(order_books, pair, balances, need_base=0.0, need_quote=(total_balances[base] * rate - total_balances[quote]) / 2)

    except Exception as e:
        Time = datetime.datetime.utcnow()
        EventType = "Error"
        Function = "rebalance"
        Explanation = "Fail while getting rates for currencies"
        EventText = e
        ExceptionType = type(e)
        print("{}|{}|{}|{}|{}|{}|{}".format(Time, EventType, Function, File, Explanation, EventText,
                                            ExceptionType))

    return ok, cur_pair, orders


# This script uses Bayesian Optimization to navigate in the face Space.
# It allows to add a figure (in folder 'new_face') in order to add them
# in real time to the space.

import numpy as np
import os
import sys
import glob
import time
import argparse
import joblib

from PIL import Image
from sklearn.gaussian_process.kernels import Matern, WhiteKernel
from utils.bayesian_optimization import Bayesian_Optimization, UtilityFunction

from utils.utils import plot_gp
if not os.path.exists('stimulus/'):
    os.makedirs('stimulus/')

parser = argparse.ArgumentParser(description="Inputs for the formatting of the Bayesian Optimizer.")
# parser.add_argument('-i', dest='init_points', type=int, nargs=1, default=[5],
#                     help="pass the number of random points sampled from the space.")
parser.add_argument('-n', dest='n_iter', type=int, nargs=1, default=[15],
                    help='pass the number of iterations for the Optimizer to run.')
parser.add_argument('-d', dest='face_display', type=bool, nargs='?', default=True,
                    help='Display faces sampled by the Optimizer.')
# parser.add_argument('-f', dest='save_file', type=bool, nargs='?', default=True,
#                     help='Name of Log file to save iteration data.')
parser.add_argument('-v', dest='vis', type=bool, nargs='?', default=True,
                    help='Show 2D and 3D visualization of the algorithm\'s progress.')
parser.add_argument('-s', dest='sim', type=bool, nargs='?',
                    help='Simulate target function.')

####################################################################################
# Example
# python3 1dBO_optm_signal.py -n 5 -d False -v True -s True
# python3 1dBO_optm_signal.py -s True
# python3 1dBO_optm_signal.py
#####################################################################################

def optm_signal(n_iter=[15], face_display=True, vis=False,
 sim=False):
    """
    This function runs the Bayesian Optimisation while it registers target values from file n170output.txt

    :type init_points: int
    :type n_iter:  int
    :type face_display: bool
    :type save_file: bool
    :type sim: bool
    :param init_points: number of points sampled randomly before the algorithm is run
    :param n_iter:  number of points samples by the algorithm
    :param face_display: if the image correspondent to the point sampled is displayed or not
    :param save_file: if a file with the iteration results is saved or not
    :param sim: if the target function is simulated or not.
    :return:
    """

    try:
        n_iter = n_iter[0]
        np.random.seed(99)
        utility_function = 'ei'

        filename = 'Subj_ID.txt'
        lines = open(filename).read().splitlines()
        try:
            save_file = str(lines[-1])
        except IndexError:
            print("Failed to read participant name. Will save data in participant_x")
            save_file = 'participant_x'

        log_dir = 'data/'+ save_file + '/'
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)
        log_name = '1dbo_' + str(int(round(time.time())))
        LOG_PATH = log_dir+log_name+'/'

        if face_display:
            # Change to Path?
            images_path = sorted(list(glob.glob('1dBO/images/*.png')))  # PATH TO 9 FACES

        space = np.array([-1, -0.8, -0.6, -0.4, -0.2, 0, 0.2 ,0.4, 0.6, 0.8, 1.0, 1.2])

        if sim:
            def blackbox_function(x):
                return -x ** 2 + 1 + np.random.normal(0,0.1,1)[0]
        else:
            def blackbox_function(x):

                # Reading the magnitude of the N170 data
                filename = './Output.txt'
                lines = open(filename).read().splitlines()

                try:
                    latency = float(lines[-1])
                except ValueError:
                    print('Failed to convert value to float')
                    wait = input("PRESS ENTER TO CONTINUE.")
                    lines = open(filename).read().splitlines()
                    latency = float(lines[-1])
                except IndexError:
                    wait = input("PRESS ENTER TO CONTINUE.")
                    lines = open(filename).read().splitlines()
                    latency = float(lines[-1])

                return latency

        pbounds = {'x': (-1, 1.2)}
        optimizer = Bayesian_Optimization(
            pbounds=pbounds,
            random_state=42,
        )
        kernel = Matern(length_scale=0.2, length_scale_bounds=(1e-1, 1e1), nu=1.5) + WhiteKernel(noise_level=0.5)
        optimizer._gp.set_params(kernel=kernel, normalize_y=True)

        # 2 Utility functions w different acquisition functions w parameters to optimize exploration
        if utility_function == 'ei':
            utility = UtilityFunction(function="ei", hyperparam=1e-1) #0 to 0.1
        elif utility_function == 'ucb':
            # check hps
            utility = UtilityFunction(function="ucb", hyperparam=1) # 1 to 10 - bigger more exploratory

        prev_latency = 0
        warm_start = np.array([1.2, -1, 0.2, -0.2])
        init_points = warm_start.shape[0]

        if not os.path.exists(LOG_PATH):
            os.makedirs(LOG_PATH)

        with open(LOG_PATH + 'log.txt', "w+") as f:
            f.write('|   iter    |  target   |     x     |     y     |\n')
            f.write('-------------------------------------------------\n')

            for it in range(init_points + n_iter):
                if it < init_points:
                    next_point_to_probe = {'x': warm_start[it]}

                    print("Next point to probe is:", next_point_to_probe)

                    x = list(next_point_to_probe.values())
                else:
                    next_point_to_probe = optimizer.suggest(utility,
                                                            n_warmup=1000,
                                                            n_iter=100)

                    print("Next point to probe is:", next_point_to_probe)

                    x = list(next_point_to_probe.values())

                for i in range(0, 10):
                    #step = i * 0.01
                    step = i * 0.1
                    close_points_idx = np.where((space[:] > x[0] - step) & (space[:] < x[0] + step))
                    if close_points_idx[0].size > 0:  # Check if not empty
                        break

                # If more than one point identified, use euclidean distance to choose best
                a = np.array(x)
                dist = []
                idx_dist = 0

                for i in range(0, close_points_idx[0].size):
                    b = space[close_points_idx[0][i]]
                    tmp = np.linalg.norm(a - b)
                    dist.append(tmp)
                if len(dist) > 1:
                    idx_dist = np.where(dist == np.min(dist))
                print(f"Closest point: {space[close_points_idx[0][i]]}")

                idx_x = np.where(space[:] == space[close_points_idx[0][i]])[0]
                print(f"Number of sampled image: {idx_x}")
                # print('Real image position: {}'.format(b))
                print('Sampled position: {}'.format(a))
                # print('Distance: {}'.format(np.min(dist)))

                if face_display:
                    # identify image associated with closest point and save as new stimulus
                    try:
                        image_path = images_path[idx_x]
                    # when there is only 1 value, idx is returned as an array
                    except:
                        image_path = images_path[idx_x[0]]
                    img = Image.open(image_path)
                    img.save('stimulus/ganstim01.png')
                    img.show()

                target = blackbox_function(**next_point_to_probe)
                # 999 is the break signal from matlab script
                if target == 999:
                    break

                while target == prev_latency or target == 0:
                    time.sleep(1)
                    print('Waiting for update')
                    target = blackbox_function(**next_point_to_probe)
                prev_latency = target
                print("Found the target value to be: ", target)
                print("________________________________________")

                if sim:
                    # register_sample = {'x': b[0]}
                    register_sample = next_point_to_probe
                else:
                    register_sample = {'x': b} # The value registered is the existing point closest to the chosen
                try:
                    optimizer.register(
                    params=register_sample,
                    target=target,
                )
                except KeyError:
                    register_sample['x'].update(register_sample['x']+np.random.normal(0, 0.01, size=1)[0])
                    print("Data point is not unique")
                    optimizer.register(
                        params=register_sample,
                        target=target,
                    )

                if vis:
                    x = np.linspace(-1.1, 1.3, 50).reshape(-1, 1)
                    plot_gp(optimizer, LOG_PATH, it, utility, pbounds, x)

                if save_file:
                    f.write('|     {}     |   {:.3f}   |   {:.3f}   |\n'.format(
                        it, target, register_sample['x']))
                    f.write('----------------------------------------\n')

                if optimizer._params.shape[0] > 2:
                    if optimizer._params[-1] == optimizer._params[-2] and optimizer._params[-1] == optimizer._params[-3]:
                        break

            print("Maximum obtained by the algorithm:")
            print(optimizer.max)

            if save_file:
                f.write('Maximum obtained by the algorithm: x = {:.3f}'.format(optimizer.max['params']['x']))
        joblib.dump(optimizer, LOG_PATH+'optimizer.joblib')

    except KeyboardInterrupt as e:
        print(e)
        print("Maximum obtained by the algorithm:")
        print(optimizer.max)

        # f.write('Maximum obtained by the algorithm: x = {:.3f}'.format(optimizer.max['params']['x']))
        joblib.dump(optimizer, LOG_PATH+'optimizer.joblib')
        return
    return

if __name__ == '__main__':
    args = parser.parse_args(sys.argv[1:])
    optm_signal(n_iter=args.n_iter, face_display=args.face_display,
                vis=args.vis, sim=args.sim)


import datetime

# CAST DATE STRING INTO DATETIME
def parse_date(date_string):
    if date_string is None:
        return None
    split_date = date_string.split('-')
    parsed_date = [int(n) for n in split_date]
    return datetime.datetime(*parsed_date)

# REFORMAT PHONE NUMBER STRING FROM '(xxx) xxx-xxxx' to 'xxx-xxx-xxxx'
def parse_phone(phone_number):
    split = phone_number.split(' ')
    split[0] = split[0][1:4]
    return '-'.join(split)

# REFORMAT OCDID STRING TO URL ENCODING && REPLACE ALL INVALID CHARS
def parse_ocdid(ocdid):
    ocdid = ['%2F' if char == '/' else char for char in ocdid]
    ocdid = ['%3A' if char == ':' else char for char in ocdid]
    return ''.join(ocdid)


from django.core.management import BaseCommand
import time
from core.models import ProductPicture, Product, Vendor
import requests
import tempfile
from django.core import files
import concurrent.futures
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.files.base import ContentFile
from PIL import Image
from io import StringIO, BytesIO

"""
python manage.py system_warmup -----> call all command
pip install psutil
pip install meliae
pip install guppy3
"""

class Command(BaseCommand):
    help = "this is a necessary command if its the first time you run project" \
           "this will setup picture and slug and etc"

    @staticmethod
    def make_thumbnail(image, size=(100, 100)):
        """Makes thumbnails of given size from given image"""

        im = Image.open(image)

        im.convert('RGB')  # convert mode

        im.thumbnail(size)  # resize image

        thumb_io = BytesIO()  # create a BytesIO object

        im.save(thumb_io, 'webp', quality=85)  # save image to BytesIO object

        thumbnail = files.File(thumb_io, name=image.name)  # create a django friendly File object

        return thumbnail

    def download(self, obj):
        response = requests.get(obj.picture_src, stream=True)

        if response.status_code != requests.codes.ok:
            print(response)

        file_name = obj.picture_src.split('/')[-1]

        lf = tempfile.NamedTemporaryFile()

        for block in response.iter_content(1024 * 8):

            if not block:
                break

            lf.write(block)

        image_obj_small = files.File(lf)
        image_obj_large = files.File(lf)
        image_obj_medium = files.File(lf)
        image_obj_small.name = file_name + '_s'
        image_obj_small.name = '{}_{}.{}'.format(file_name, 's', 'webp')
        image_obj_large.name = file_name + '_l'
        image_obj_medium.name = file_name + '_m'

        small_img = self.make_thumbnail(image_obj_small, size=(100, 100))
        thumbnail = self.make_thumbnail(image_obj_medium, size=(128, 128))
        large_img = self.make_thumbnail(image_obj_large, size=(216, 216))

        obj.large = large_img
        obj.small = small_img
        obj.medium = thumbnail
        obj.save()

        print(f' image downloaded url => {obj.picture_src}')

    def initial_setup(self):

        start = time.perf_counter()

        # Product.objects.filter(pictures__state='need_resize')

        pps = list(ProductPicture.objects.filter(state='need_resize', product__vendor__name='digikala')[:50])
        # pps = list(ProductPicture.objects.filter(state='need_resize').values_list('picture_src', flat=True)[:50])
        print(f'you have {len(pps)} not downloaded image')

        with concurrent.futures.ThreadPoolExecutor() as executor:
            executor.map(self.download, pps)

        finish = time.perf_counter()

        # Time Analyze

        print(f'Finished in {round(finish-start, 2)} seconds')

    def handle(self, *args, **options):
        self.initial_setup()


from typing import Any, Optional

import pytest
from fastapi.testclient import TestClient

from merino.main import app
from merino.providers import get_providers
from tests.unit.web.util import filter_caplog
from tests.unit.web.util import get_providers as override_dependency

client = TestClient(app)
app.dependency_overrides[get_providers] = override_dependency

# The first two IP addresses are taken from `GeoLite2-City-Test.mmdb`
@pytest.mark.parametrize(
    "ip,expected_country,expected_region,expected_city,expected_dma",
    [
        ("216.160.83.56", "US", "WA", "Milton", 819),
        ("2.125.160.216", "GB", "ENG", "Boxford", None),
        ("127.0.0.1", None, None, None, None),
    ],
)
def test_geolocation(
    mocker: Any,
    caplog: Any,
    ip: str,
    expected_country: Optional[str],
    expected_region: Optional[str],
    expected_city: Optional[str],
    expected_dma: Optional[int],
) -> None:
    import logging

    caplog.set_level(logging.INFO)

    mock_client = mocker.patch("fastapi.Request.client")
    mock_client.host = ip

    client.get("/api/v1/suggest?q=nope")

    records = filter_caplog(caplog.records, "web.suggest.request")

    assert len(records) == 1

    record = records[0]

    assert record.__dict__["country"] == expected_country
    assert record.__dict__["region"] == expected_region
    assert record.__dict__["city"] == expected_city
    assert record.__dict__["dma"] == expected_dma

def test_geolocation_with_invalid_ip(mocker: Any, caplog: Any) -> None:
    import logging

    caplog.set_level(logging.WARNING)

    mock_client = mocker.patch("fastapi.Request.client")
    mock_client.host = "invalid-ip"

    client.get("/api/v1/suggest?q=nope")

    records = filter_caplog(caplog.records, "merino.middleware.geolocation")

    assert len(records) == 1
    assert records[0].message == "Invalid IP address for geolocation parsing"

def test_geolocation_with_none_ip(mocker: Any, caplog: Any) -> None:
    import logging

    caplog.set_level(logging.WARNING)

    mock_client = mocker.patch("fastapi.Request.client")
    mock_client.host = None

    client.get("/api/v1/suggest?q=nope")

    records = filter_caplog(caplog.records, "merino.middleware.geolocation")

    assert len(records) == 1
    assert records[0].message == "Invalid IP address for geolocation parsing"

def test_geolocation_with_no_client(mocker: Any, caplog: Any) -> None:
    import logging

    caplog.set_level(logging.WARNING)

    mock_req = mocker.patch("fastapi.Request")
    mock_req.client = None

    client.get("/api/v1/suggest?q=nope")

    records = filter_caplog(caplog.records, "merino.middleware.geolocation")

    assert len(records) == 1
    assert records[0].message == "Invalid IP address for geolocation parsing"


#!/usr/bin/env python
import os, time, json
from re import match, split
from tqdm import tqdm
from argparse import ArgumentParser
# https://dboyliao.medium.com/python-%E8%B6%85%E5%A5%BD%E7%94%A8%E6%A8%99%E6%BA%96%E5%87%BD%E5%BC%8F%E5%BA%AB-argparse-4eab2e9dcc69

argp = ArgumentParser()
argp.add_argument("-s", "--source", help="Data source", dest="datasource")
argp.add_argument("-p", "--pattern", help="Filtering with Regex", dest="pattern", default=".*")
argp.add_argument("-c", "--encoding", help="File encoding", dest="coded", default="utf-8")
# https://realpython.com/working-with-files-in-python/
args = argp.parse_args()
_rp = args.pattern
_ds = args.datasource
_coded = args.coded
dslist = os.listdir(_ds)

# filtering https://realpython.com/working-with-files-in-python/
# https://stackoverflow.com/questions/3640359/regular-expressions-search-in-list
# Testing https://regex101.com/
vocab = set()
keep = list(filter(lambda v: match(_rp, v), dslist))
for doc in tqdm(keep):
  doc_path = f'{_ds}/{doc}'
  # https://stackoverflow.com/questions/10971033/backporting-python-3-openencoding-utf-8-to-python-2
  tokens = split('\s', open(doc_path, 'r', encoding=_coded, errors='replace').read())
  for token in tokens:
    vocab.add(token)

maps = {index:word for index, word in enumerate(vocab)}
# https://www.geeksforgeeks.org/get-current-timestamp-using-python/
output = open(f"_output/{time.time()}.out", "w")
output.write(json.dumps(maps))


from __future__ import print_function

import json
from optparse import make_option

from django.core.management.base import NoArgsCommand, CommandError

from disqus.api import DisqusClient

class Command(NoArgsCommand):
    option_list = NoArgsCommand.option_list + (
        make_option('--indent', default=None, dest='indent', type='int',
            help='Specifies the indent level to use when pretty-printing output'),
        make_option('--filter', default='', dest='filter', type='str',
            help='Type of entries that should be returned'),
        make_option('--exclude', default='', dest='exclude', type='str',
            help='Type of entries that should be excluded from the response'),
    )
    help = 'Output DISQUS data in JSON format'
    requires_model_validation = False

    def handle(self, **options):
        from django.conf import settings

        client = DisqusClient()
        indent = options.get('indent')
        filter_ = options.get('filter')
        exclude = options.get('exclude')

        # Get a list of all forums for an API key. Each API key can have
        # multiple forums associated. This application only supports the one
        # set in the DISQUS_WEBSITE_SHORTNAME variable
        forum_list = client.get_forum_list(user_api_key=settings.DISQUS_API_KEY)
        try:
            forum = [f for f in forum_list\
                     if f['shortname'] == settings.DISQUS_WEBSITE_SHORTNAME][0]
        except IndexError:
            raise CommandError("Could not find forum. " +
                               "Please check your " +
                               "'DISQUS_WEBSITE_SHORTNAME' setting.")
        posts = []
        has_new_posts = True
        start = 0
        step = 100
        while has_new_posts:
            new_posts = client.get_forum_posts(
                user_api_key=settings.DISQUS_API_KEY,
                forum_id=forum['id'],
                start=start,
                limit=start+step,
                filter=filter_,
                exclude=exclude)
            if not new_posts:
                has_new_posts = False
            else:
                start += step
                posts.append(new_posts)
        print(json.dumps(posts, indent=indent))


import cv2
import random
import math
import numpy as np

def augment_img(image2aug, method_selector=-1):
    #Applies one of three methods of image augmentation and returns the augmented image

	'''
		method_selector options:
		1 - perspective transformation
		2- image translation
		3- image rotation
	'''
	if (method_selector == -1):
		#random msg to select which selector to use
		method_selector = random.randint(1,3)

	image_shape = image2aug.shape

	if(method_selector == 1):
		#Apply prespective transfromation

		#getting random vertices of the image
		x1 = random.randint(0, math.ceil(image_shape[0]/8))
		x2 = random.randint(math.ceil(7 * image_shape[0]/8), image_shape[0])
		y1 = random.randint(0, math.ceil(image_shape[1]/8))
		y2 = random.randint(math.ceil(7 * image_shape[1]/8), image_shape[0])

		#applying prespective transformation
		pts1 = np.float32([[x1,y1],[x2,y1],[x1,y2],[x2,y2]])
		pts2 = np.float32([[0,0],[image_shape[0],0],[0,image_shape[1]],[image_shape[0],image_shape[1]]])
		M = cv2.getPerspectiveTransform(pts1,pts2)
		output_img = cv2.warpPerspective(image2aug,M,(image_shape[0],image_shape[1]))

	elif(method_selector == 2):
		#apply translation

		#getting random shifts for the image
		shift_x = random.randint(-math.ceil(image_shape[0]/8), math.ceil(image_shape[0]/8))
		shift_y = random.randint(-math.ceil(image_shape[1]/8), math.ceil(image_shape[1]/8))

		#applying translation
		M = np.float32([[1,0,shift_x],[0,1,shift_y]])
		output_img = cv2.warpAffine(image2aug,M,(image_shape[0],image_shape[1]))

	elif(method_selector == 3):
		#apply rotation

		#obtaining random angle to rotate
		angle2rotate = random.randint(-5, 5)

		#applying rotation
		M = cv2.getRotationMatrix2D((math.ceil(image_shape[0]/2),math.ceil(image_shape[1]/2)),angle2rotate,1)
		output_img = cv2.warpAffine(image2aug,M,(image_shape[0],image_shape[1]))

	return output_img

def rgb2gray(rgb_list):
	#Converts a list of rgb images to grayscale

	if (type(rgb_list) is not list) and (type(rgb_list) is not np.ndarray):
		rgb_list = [rgb_list]

	imgGray_list = []

	for img in rgb_list:
		grayCh = np.zeros([img.shape[0], img.shape[1], 1])
		grayCh[:,:,0] = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
		imgGray_list.append(grayCh)

	return imgGray_list

def normalize(img_list, mean=128, std=128):
    #normalizes the pixel values of a list of images

	if (type(img_list) != list) and (type(img_list) != np.ndarray):
		img_list = [img_list]

	norm_list = []

	for img in img_list:
		norm_list.append(np.divide(img - mean, std))

	return norm_list


def allcard(xdata,zdata,xval,zval,**kwargs):
    # enumerate all model cardinalities via ccmiqp and
    # use validation/cross-validaiton to determine
    import alamopy
    from alamopy.writethis import writethis
    import numpy as np
    import math
    import sympy
    import random
    from random import shuffle
    from scipy.optimize import curve_fit #2.7
    from scipy.optimize import minimize #2.7
    from scipy.linalg import lstsq # 2.7
    import os
    from alamopy import almlsq as almfun
    from alamopy import almlsqjac as almjac
    import time
    from sympy.parsing.sympy_parser import parse_expr
    import sys

    random.seed(100)

    # kwargs=opdict
    trans=list(['linfcns','expfcns','logfcns','sinfcns','cosfcns','monomialpower','multi2power','multi3power','ratiopower'])
    et = list(['multi2power','multi3power','ratiopower'])
    datacc = {}
    ins = list(['cvfold'])
    for opt in ins:
        if (opt in kwargs.keys()):
            cvfold=kwargs['cvfold']
    ntrans=0
    ndata = np.shape(xdata)[0]
    ninputs = np.shape(xdata)[1]

    for t in trans:
        if t in kwargs.keys():
            if (type(kwargs[t]) == list()):
                nt = len(kwargs[t])
            else:
                nt=1
            if t not in et:
                ntrans=ntrans+ninputs*nt
            else:
                ntrans=ntrans+math.factorial(ninputs)*nt
        else:
            kwargs[t]=0
    mseold = 0.0
    rmseold = 0.0
    startt = time.time()
    oldp = ()
    ntrans = min(ntrans,1000)

    # split trianing and validationd ata before looping through cc
    tlist=list([])
    vlist=list([])
    ndlist = range(ndata)
    if (cvfold == 'loo'):
        for i in range(ndata):
            vlist.append(np.asarray([i]))
            temp = [x for x in range(ndata) if x != i]
            tlist.append(np.asarray(temp))
    else:
        temp = range(ndata)
        shuffle(temp)
#            tlist = np.asarray(temp)
        if (cvfold == 'valset'):
            tlist = np.asarray(temp)[0:int(0.7*ndata)]
            vlist = np.asarray(temp)[int(0.7*ndata):-1]
        else:
            vlist = np.array_split(np.asarray(temp),int(cvfold))
            tlist = [1]*int(cvfold)
            for v in range(len(vlist)):
                tlist[v] = range(ndata)
                for this in vlist[v]:
                    tlist[v].remove(this)

    for ccon in list(range(1,ntrans+1)):
        reload(alamopy)
        try:
            del cvalsim, almsim
        except:
            pass

#        res = alamopy.doalamo(xdata,zdata,kwargs.values())
        if (cvfold != 'valset'):
            res=alamopy.doalamo(xdata,zdata,xval=xval,zval=zval,linfcns=kwargs['linfcns'],expfcns=kwargs['expfcns'],logfcns=kwargs['logfcns'],sinfcns=kwargs['sinfcns'],cosfcns=kwargs['cosfcns'],monomialpower=kwargs['monomialpower'],multi2power=kwargs['multi2power'],multi3power=kwargs['multi3power'],ratiopower=kwargs['ratiopower'],sigma=kwargs['sigma'],xlabels=kwargs['xlabels'],zlabels=kwargs['zlabels'],modeler=6,convpen=0,maxterms=ccon,almname=kwargs['almname'],expandoutput=kwargs['expandoutput'],xmax=kwargs['xmax'],xmin=kwargs['xmin'],savescratch=kwargs['savescratch'])
        else:
            res=alamopy.doalamo(xdata[tlist,:],zdata[tlist],xval=xdata[vlist,:],zval=zdata[vlist],linfcns=kwargs['linfcns'],expfcns=kwargs['expfcns'],logfcns=kwargs['logfcns'],sinfcns=kwargs['sinfcns'],cosfcns=kwargs['cosfcns'],monomialpower=kwargs['monomialpower'],multi2power=kwargs['multi2power'],multi3power=kwargs['multi3power'],ratiopower=kwargs['ratiopower'],sigma=kwargs['sigma'],xlabels=kwargs['xlabels'],zlabels=kwargs['zlabels'],modeler=6,convpen=0,maxterms=ccon,almname=kwargs['almname'],expandoutput=kwargs['expandoutput'],xmax=kwargs['xmax'],xmin=kwargs['xmin'],savescratch=kwargs['savescratch'])
        for fn in ['cvalsim.py','cvalsim.pyc','almsim.py','almsim.pyc']:
            try:
                os.remove(fn)
            except:
                pass
        os.system('cp '+kwargs['almname'].split('.')[0]+'cv.py cvalsim.py')
        os.system('cp '+kwargs['almname'].split('.')[0]+'alm.py almsim.py')

        if ( ccon == 1):
            if (ndata < 10): #not enough data to do cross validation
                writethis('Not enough data to facilitate cross validation')
                endt = time.time()
                res['totaltime']= endt - startt
                return res

        import cvalsim
        import almsim
        xalm = alamopy.almfeatmat(xdata,ccon)
        # Lets do the cross validatione error
        mseval=0.0
        rmse = {}
        if (cvfold == 'valset'):
            resid=np.sum((zdata[vlist] - almsim.f(xdata[vlist,:]))**2)
            mseval = resid / float(len(vlist))
            params = 'ALM params used for valset'
            rmse={}
            rmse['val'] = res['rmseval']
            rmse['train'] =res['rmse']
        else:
            mseval = 0.0
            track = 0
            for tl, vl in zip(tlist,vlist):
                initb = [ 0.0 for x in range(ccon)]
                xd,xmax,xmin=alamopy.mapminmax(xalm)
                zd,zmax,zmin=alamopy.mapminmax(zdata)
                fitres = lstsq(xd[tl],zd[tl])
                params = fitres[0]
                resid = zd[vl] - np.matmul(xd[vl,:],params[:])
                resid = alamopy.remapminmax(resid,zmax,zmin)
                resid = sum(resid ** 2)
                if (cvfold == 'loo'):
                    mseval = mseval + resid
                else:
                    mseval = mseval + resid/float(len(vl))
            mseval = mseval / float(len(vlist))
            rmse['val'] = np.sqrt(mseval)
            rmse['train'] = 'not'
        if ( ccon > 1):
            if (float(rmse['val']) >= float(rmseold)):
                sys.stdout.write('              Problem name   : '+kwargs['almname']+'\n')
                sys.stdout.write('  rMSEval : '+str(rmse['val'])+'    MSEold : '+str(rmseold)+'\n')
                if ( cvfold == 'valset'):
                    sys.stdout.write('   Ntrain : '+str(len(tlist))+'    Nval : '+str(len(vlist))+'\n')
                else:
                    sys.stdout.write('   Ntrain : '+str(len(tlist[0]))+'    Nval : '+str(len(vlist[0]))+'\n')
                sys.stdout.write('       optimal model size is : '+str(ccon-1)+'\n')
                sys.stdout.write('    optimal coefficients are : '+str(oldp)+'\n')
                os.remove(kwargs['almname'].split('.')[0]+'alm.py')
                os.system('cp '+'oldalmsim.sv '+kwargs['almname'].split('.')[0]+'alm.py')
                endt = time.time()
                oldres['totaltime']= endt - startt
                return oldres
            elif (ccon == ntrans):
                endt = time.time()
                sys.stdout.write('optimal model size is :'+str(ccon)+'\n')
                res['totaltime'] = endt - startt
                return res
#            else:
#                mseold=mseval
#                oldres=res
#        else:
        mseold=float(mseval)
        rmseold = float(rmse['val'])
        oldres=res
        oldp=params
        # keep track of alm model of old iteratoin
        try:
            os.remove('oldalmsim.sv')
        except:
            pass
        os.system('mv almsim.py oldalmsim.sv')

def almlsq(params,*X):
    #return lsq objective
    import numpy as np
    try:
        reload(cvalsim)
    except:
        import cvalsim
    from cvalsim import f as sim

    xdata,zdata = X[0]
    return sum((zdata - sim(xdata,params))**2)

def almlsqjac(params,*X):
    try:
        reload(cvalsim)
    except:
        import cvalsim
    from cvalsim import f as sim
    import numpy as np

    xdata,zdata=X[0]
    rp = np.ones(len(params))
    for i in range(len(params)):
        dparams = [0.0 for x in range(len(params))]
        dparams[i] = 1.0
        rp[i] = sum(-2.0*(zdata-sim(xdata,params))*sim(xdata,dparams))
    return rp
    #return np.dot(params,np.matmul(np.transpose(xdata),xdata))+np.matmul(zdata,xdata)

def almfeatmat(X,nparams):
    import numpy as np
    try:
        reload(cvalsim)
    except:
        import cvalsim
    from cvalsim import f as sim

    dims = np.shape(X)
    fm = np.ones([dims[0],nparams])
    dparams = [0.0 for x in range(nparams)]
    for i in range(nparams):
        dparams[i] = 1.0
        fm[:,i] = sim(X,dparams)
        dparams[i] = 0.0
    return fm