# Image Process

try:
    import boto3
    from botocore.exceptions import ClientError
    import botocore
    import os
    import sys
    import json
except Exception as e:
    print(e)
import time
import logging
import PIL.Image
from logging.handlers import RotatingFileHandler

# Variable global pour les noms des queues d'entrée et de sortie.
AWS_SQS_QUEUE_NAME_INPUT = "Inbox"
AWS_SQS_QUEUE_NAME_OUTPUT = "Outbox"
AWS_S3_BUCKET ="mybucket307991"

#Class de type Client.
class client(object):

    def __init__(self):
        self.flag=0
        try:
            # Create the queue. This returns an SQS.Queue instance
            self.inputqueue = sqs.create_queue(QueueName=AWS_SQS_QUEUE_NAME_INPUT)
            self.receivedKeyName = None  #The key Name of the processed Image
        except Exception:
            print("Connexion Failed !")

        # Get the queue
        self.inputqueue = sqs.get_queue_by_name(QueueName=AWS_SQS_QUEUE_NAME_INPUT)
        print("You are connected to the SQS service ...")

    def upload_image(Self, file_name, bucket, key_name):
        s3 = boto3.client('s3')
        s3.upload_file(file_name, bucket, key_name)

        # Upload the image
        try:
        	response = s3.upload_file(file_name, bucket, key_name)
        	print("The image is sent !!")
        except ClientError as e:
            logging.error(e)
        # return False

	#Download the image
    def download_image(self, bucket, key_name, local_name):
        s3 = boto3.resource('s3')
        try:
            s3.Bucket(bucket).download_file(key_name,local_name)
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == "404":
                print("The object does not exist.")
            else:
                raise
    #Send message
    def send(self, message=None):
        # Create a new message
        response = self.inputqueue.send_message(MessageBody=message)
        print("The Key of your image is sent !!")
        return response

    # Receive the Key Name of the processed Image from the sqs (OutboxQueue)
    def receive(self):
        try:
            self.queue = sqs.get_queue_by_name(QueueName=AWS_SQS_QUEUE_NAME_OUTPUT)
            for message in self.queue.receive_messages():
                data = message.body
                print("Here is the name of your processed image : ",str(data),".jpg, check the directory from where you started your client program :)\n")
                self.receivedKeyName = str(data)
                #Download the processed Image by his Key Name
                self.download_image(AWS_S3_BUCKET, str(q.receivedKeyName), str(q.receivedKeyName)+".jpg")
                processedImageDisplay = PIL.Image.open(str(data)+".jpg")
                processedImageDisplay.show()
                message.delete()

                self.flag=1
        except Exception:

            return []

if __name__ == "__main__":
    # Get the service resource
    sqs = boto3.resource('sqs')
    print("You are connected to the AWS service ...")
    flag = 0
    q = client()
    while True:
        if flag == 0 :
            command = int(input("Hi, Dear Client choose one option Tap : \n"
                         "1) For sending the key of your image that you want to proccess \n"
                         "2) To exit\n"))
            flag=flag+1

        else :
            command = int(input("Hi again, Dear Client choose one option Tap : \n"
                         "1) For sending the key of your image that you want to proccess \n"
                         "2) To exit\n"))

        if command == 1 :
            ImageName = input("Enter your image name (that exist in same directory of the program) : ")
            keyName = input("Enter your key image (that exist in same directory of the program) : ")
            q.upload_image( ImageName, AWS_S3_BUCKET, keyName)
            response = q.send(message=keyName)
            while q.flag==0:
                time.sleep(0.5)
                print("Waiting for the processed Image ...")
                #Receive the Key name of the Key Name from Outbox Queue and saved in "q.receivedKeyName" self variable .
                q.receive()
            q.flag=0
        elif command == 2:
            print("See you, Bye!")
            exit()


# coding=utf-8

#     National Oceanic and Atmospheric Administration (NOAA)
#     Alaskan Fisheries Science Center (AFSC)
#     Resource Assessment and Conservation Engineering (RACE)
#     Midwater Assessment and Conservation Engineering (MACE)

# THIS SOFTWARE AND ITS DOCUMENTATION ARE CONSIDERED TO BE IN THE PUBLIC DOMAIN
# AND THUS ARE AVAILABLE FOR UNRESTRICTED PUBLIC USE. THEY ARE FURNISHED "AS
# IS. THE AUTHORS, THE UNITED STATES GOVERNMENT, ITS INSTRUMENTALITIES,
# OFFICERS, EMPLOYEES, AND AGENTS MAKE NO WARRANTY, EXPRESS OR IMPLIED,
# AS TO THE USEFULNESS OF THE SOFTWARE AND DOCUMENTATION FOR ANY PURPOSE.
# THEY ASSUME NO RESPONSIBILITY (1) FOR THE USE OF THE SOFTWARE AND
# DOCUMENTATION; OR (2) TO PROVIDE TECHNICAL SUPPORT TO USERS.

"""

| Developed by:  Rick Towler   <rick.towler@noaa.gov>
| National Oceanic and Atmospheric Administration (NOAA)
| Alaska Fisheries Science Center (AFSC)
| Midwater Assessment and Conservation Engineering Group (MACE)
|
| Author:
|       Rick Towler   <rick.towler@noaa.gov>
| Maintained by:
|       Rick Towler   <rick.towler@noaa.gov>

"""

import numpy as np
from ..ping_data import ping_data

class line(ping_data):
    #   TODO: Review attributes in this docstring
    """The line class implements lines based on ping_time and depth/range values.
    The class provides methods manipulating these values in various ways. The
    numerical operators operate on the data, allowing offset lines to easily
    be created. Color and name properties can be used in plotting the lines.

    Attributes:
        ping_time: ping_time is a datetime object that defines the time the
            ping was recorded.
        data: data is a numpy array which contains the float data.
        color: color is a list which defines the color of the line.
        name (string): name or label for the line is a string.
        linestyle: linestyle is a string that defines the style of the line.
        thickness: thickness is a float the defines the width of the line.
    """

    def __init__(self, ping_time=None, data=None, color=[0.58, 0.0, 0.83],
                 name='line', linestyle='solid', thickness=1.0, **_):
        """Initializes line class object.

        Creates and sets several internal properties.
        """
        super(line, self).__init__()

        # Set the ping time.
        self.ping_time = ping_time

        # Set the number of pings.
        if ping_time is not None:
            self.n_pings = ping_time.shape[0]

        # Assign data based on what we're given. Arrays must be the same shape
        # as ping_time, scalars are expanded to the same shape as ping_time
        # and None is None.
        if isinstance(data, np.ndarray):
            if data.ndim == 0:
                data = np.full(ping_time.shape[0], data, dtype='float32')
            else:
                if data.shape[0] != ping_time.shape[0]:
                    raise ValueError("The data array must be None, a scalar "
                                     "or an array the same size as ping_time.")
        elif data is not None:
            try:
                data = float(data)
                data = np.full(ping_time.shape[0], data, dtype='float32')
            except Exception:
                raise ValueError("The data array must be None, a scalar or an"
                                 " array the same size as ping_time.")
        self.data = data

        # Set the initial attribute values.
        self.color = color
        self.name = name
        self.linestyle = linestyle
        self.thickness = thickness

        # Update out data_attributes list, adding the "data" attribute.
        self._data_attributes += ['data']

    def interpolate(self, new_times):
        """Interpolates the data values to the provided time vector.

        Interpolate interpolates the data values to the provided ping_data derived
        object or a vector of datetime64 times.

        Args:
            new_times (ping_data or array): an instance of a ping_data object
                (raw_data, processed_data) OR a 1D numpy array of new dateTime64
                times
        """

        #  check if the times are to be grabbed from a ping_data object
        if isinstance(new_times, ping_data):
            new_times = new_times.ping_time

        #  and interpolate
        self.data[:] = np.interp(self.ping_time, new_times,
                    self.data, left=np.nan, right=np.nan)
        self.ping_time = new_times.copy()

    def _setup_numeric(self, other):
        """Internal method containing generalized numeric operators code.

         _setup_numeric is an internal method that contains generalized code for
        the numeric operators. Biggest job is interpolating ping times if the
        current line instance has a different number of pings from "other".

        Args:
            other(line object or array or scalar value): The data that is
            being used in the operator along with the line instance being
            operated on.

        Returns:
            The data from other if it's a line object or other if other is a
            scalar or array.
        """

        if isinstance(other, line):
            if other.data.shape[0] != self.data.shape[0]:
                # The other line has a different number of pings so
                # interpolate to this line's pings.
                other_data = np.interp(self.ping_time, other.ping_time,
                                       self.data, left=np.nan, right=np.nan)
            else:
                other_data = other.data
        else:
            # Assume this is a scalar or array that is compatible.
            other_data = other

        return other_data

    def __add__(self, other):
        """Implements the binary addition operator.

        __add__ implements the binary addition operator.

        Args:
            other(line object or array or scalar value): The data that is
                being added to the line instance being operated on.

        Returns:
            line object that is the result of adding "other" to the line object.
        """
        other_data = self._setup_numeric(other)
        new_line = empty_like(self)

        if isinstance(other, line):
            other_data = other.data
        else:
            other_data = other

        new_line.data[:] = self.data + other_data

        return new_line

    def __radd__(self, other):
        """Implements the reflected binary addition operator.

        __radd__ implements the reflected binary addition operator.

        Args:
            other(line object or array or scalar value): The data that is
                being reflected added to the line instance being operated on.

        Returns:
            line object that is the result of reflective adding "other" to
            the line object.
        """

        return self.__add__(other)

    def __iadd__(self, other):
        """Implements the in-place binary addition operator.

        __iadd__ implements the in-place binary addition operator

        Args:
            other(line object or array or scalar value): The data that is
                being added to the line instance being operated on.

        Returns:
            Returns original line object with "other" added to it.
        """

        other_data = self._setup_numeric(other)
        self.data[:] = self.data + other_data

        return self

    def __sub__(self, other):
        """Implements the binary subtraction operator.

        __sub__ implements the binary subtraction operator.

        Args:
            other(line object or array or scalar value): The data that is
                being subtracted from the line instance being operated on.

        Returns:
            New line object that is the original object minus "other".
        """

        other_data = self._setup_numeric(other)
        new_line = empty_like(self)
        if isinstance(other, line):
            other_data = other.data
        else:
            other_data = other

        new_line.data[:] = self.data - other_data

        return new_line

    def __rsub__(self, other):
        """Implements the reflected binary subtraction operator.

        __rsub__ implements the reflected binary subtraction operator.

        Args:
            other(line object or array or scalar value): The data that is
            being subtracted from the line instance being operated on.

        Returns:
            New line object that is the original object minus "other".
        """

        return self.__sub__(other)

    def __isub__(self, other):
        """Implements the in-place binary subtraction operator.

        __isub__ implements the in-place binary subtraction operator.

        Args:
            other(line object or array or scalar value): The data that is
            being subtracted from the line instance being operated on.

        Returns:
            Original line object with "other" subtracted from it.
        """
        other_data = self._setup_numeric(other)
        self.data[:] = self.data - other_data

        return self

    def __mul__(self, other):
        """Implements the binary multiplication operator.

        __mul__ implements the binary multiplication operator.

        Args:
            other(line object or array or scalar value): The data that is
            being multiplied with the line instance being operated on.

        Returns:
            New line object that is the original object multiplied by  "other".
        """

        other_data = self._setup_numeric(other)
        new_line = empty_like(self)
        if isinstance(other, line):
            other_data = other.data
        else:
            other_data = other

        new_line.data[:] = self.data * other_data

        return new_line

    def __rmul__(self, other):
        """Implements the reflected binary multiplication operator.

        __rmul__ implements the reflected binary multiplication operator.

        Args:
            other(line object or array or scalar value): The data that is
            being multiplied with the line instance being operated on.

        Returns:
            New line object that is the original object multiplied by  "other".
        """

        return self.__mul__(other)

    def __imul__(self, other):
        """Implements the in-place binary multiplication operator.

        __imul__ implements the in-place binary multiplication operator.

        Args:
            other(line object or array or scalar value): The data that is
            being multiplied with the line instance being operated on.

        Returns:
            Original line object multiplied by other".
        """

        other_data = self._setup_numeric(other)
        self.data[:] = self.data * other_data

        return self

    def __truediv__(self, other):
        """Implements the binary fp division operator.

        __truediv__ implements the binary fp division operator.

        Args:
            other(line object or array or scalar value): The data to divide the
            line instance being operated on.

        Returns:
            New line object that is the original line object divided bu other.
        """
        other_data = self._setup_numeric(other)
        new_line = empty_like(self)
        if isinstance(other, line):
            other_data = other.data
        else:
            other_data = other

        new_line.data[:] = self.data / other_data

        return new_line

    def __rtruediv__(self, other):
        """Implements the reflected binary fp division operator.

        __rtruediv__ implements the reflected binary fp division operator.

        Args:
            other(line object or array or scalar value): The data to divide the
            line instance being operated on.
        Returns:
            New line object that is the original line object divided bu other.
        """

        return self.__truediv__(other)

    def __itruediv__(self, other):
        """Implements the in-place binary fp division operator.

        __itruediv__ implements the in-place binary fp division operator.

        Args:
            other(line object or array or scalar value): The data to divide the
            line instance being operated on.

        Returns:
            Original line object divided by other.
        """

        other_data = self._setup_numeric(other)
        self.data[:] = self.data / other_data

        return self

    def __pow__(self, other):
        """Implements the binary power operator.

        __pow__ implements the binary power operator.

        Args:
            other(line object or array or scalar value): The data to raise the
            line instance being operated on.

        Returns:
            New line object that is the original line raided to "other".
        """
        other_data = self._setup_numeric(other)
        new_line = empty_like(self)
        if isinstance(other, line):
            other_data = other.data
        else:
            other_data = other

        new_line.data[:] = self.data ** other_data

        return new_line

    def __rpow__(self, other):
        """Implements the reflected binary power operator.

        __rpow__ implements the reflected binary power operator.

        Args:
            other(line object or array or scalar value): The data to raise the
            line instance being operated on.

        Returns:
            New line object that is the original line raided to "other".
        """

        return self.__pow__(other)

    def __ipow__(self, other):
        """Implements the in-place binary power operator.

        __ipow__ implements the in-place binary power operator.

        Args:
           other(line object or array or scalar value): The data to raise the
            line instance being operated on.

        Returns:
            Original line object that raided to "other".
        """

        other_data = self._setup_numeric(other)
        self.data[:] = self.data ** other_data

        return self

    def __str__(self):
        """Re-implements string method to provide basic information.

        Reimplemented string method that provides some basic info about the
        mask object.

        Return:
            A message with basic information about the mask object.
        """

        # Print the class and address.
        msg = "{0} at {1}\n".format(str(self.__class__), str(hex(id(self))))

        # Print some other basic information.
        msg = "{0}                 line name: ({1})\n".format(msg, self.name)
        msg = "{0}                 ping_time: ({1})\n".format(
                                                    msg,
                                                    self.ping_time.shape[0])
        msg = "{0}                      data: ({1})\n".format(
                                                            msg,
                                                            self.data.shape[0])
        msg = "{0}                start time: {1}\n".format(msg,
                                                            self.ping_time[0])
        msg = "{0}                  end time: {1}\n" .format(msg,
                                                             self.ping_time[-1])

        return msg

def empty_like(obj, name=None, color=None, linestyle=None, thickness=None,
    initialize=True):
    """Creates an empty line object with the same time values as the provided object.

    empty_like creates an empty line (where data values are NaN) that is the same
    length as the provided line or processed data object. If a line object is passed,
    the name and color attributes are copied if not explicitly provided. If a
    processed data object is passed the returned line will have the default line
    attributes if not explicitly provided.

    Args:
        obj (line or processed_data object): The line or processed_data object
            instance that is the template for the new object being created.
        name (str): Optional name for the new line object.
        color (): Optional color for the new line object.
        linestyle(str): Optional linestyle is a string that defines the style of the line.
        thickness(float): Optional thickness is a float the defines the width of the line.
        initialize (bool): This argument is used internally to skip intializing the data
            attribute.

    Returns:
        New empty instance of line object with name and color ether
        copied from line object passed in or using optional parameters
        passed to method.
    """
    # Create a new line object to return.
    new_line = line(ping_time=obj.ping_time.copy())

    # Check if new properties were provided, otherwise copy from original.
    if color:
        new_line.color = color
    else:
        if isinstance(obj, line):
            new_line.color = obj.color
    if name:
        new_line.name = name
    else:
        if isinstance(obj, line):
            new_line.name = obj.name
    if linestyle:
        new_line.linestyle = linestyle
    else:
        if isinstance(obj, line):
            new_line.linestyle = obj.linestyle
    if thickness:
        new_line.thickness = thickness
    else:
        if isinstance(obj, line):
            new_line.thickness = obj.thickness

    # Set the data array to NaNs.
    new_line.data = np.full(new_line.ping_time.shape[0], np.nan)

    return new_line

def like(obj, **kwargs):
    """Creates a new line object that is a copy of the provided line.

    like creates an copy of an existing line where both the ping time and data
    values are the same. The name and color attributes are copied if not
    explicitly provided.

    Args:
        obj (line object): The line object that is to be copied.
        name (str): Optional name for the new line object.
        color (): Optional color for the new line object.
        linestyle(str): Optional linestyle is a string that defines the style of the line.
        thickness(float): Optional thickness is a float the defines the width of the line.

    Returns:
        A copy of the provided line object
    """

    # Create a new line object that is the same as the provided line.
    # Set the initialize leyword to False to skip initializing the data attribute.
    new_line = empty_like(obj, initialize=False, **kwargs)

    # Set the new line data attribute to a copy of the provided line's data
    new_line.data = obj.data.copy()

    return new_line

def read_evl(evl_filename, name='evl_line', ignore_status=False, **kwargs):
    '''read_evl will read a .evl file exported by Echoview and return a line object
    containing the Echoview line data. Vertices that have a status other than 3 ("good")
    will be assigned NaN.

    evl_filename (string): The full path to the Echoview .evl file to read
    name (string): name or label for the line.
    ignore_status (bool): Set to True to ignore the .evl vertex status. Vertices in
        Echoview .evl files are assigned a status where:
            0 = no status
            1 = unverified
            2 = bad
            3 = good
        By default, status values less than 3 are assigned a value of NaN. Set this
        keyword to True to assign depth values to the vertices regardeless of status.
        Note that vertices with the special value of -10000.99 are always assigned NaN
    color: color is a list which defines the color of the line.
    linestyle: linestyle is a string that defines the style of the line.
    thickness: thickness is a float the defines the width of the line.

    '''

    import os
    from datetime import datetime

    def convert_float(val):
        try:
            val = float(val)
        except:
            val = np.nan
        return val

    # Normalize filename and read the file
    evl_filename = os.path.normpath(evl_filename)
    with open(evl_filename, 'r') as infile:
        evl_data = infile.readlines()

    # Discard the file headers
    evl_data = evl_data[2:]

    #  determine the number of line vertices
    n_pings = len(evl_data)

    # Echoview .evl files contain ping time, depth, and line status data.
    # Create the time and depth arrays. Status is used to determine if the
    # depth data is valid or a Nan is inserted instead
    depth_data = np.empty((n_pings), dtype=np.float32)
    ping_time = np.empty((n_pings), dtype='datetime64[ms]')

    # Loop thru the rows of data, parsing each line
    for idx, row in enumerate(evl_data):
        #  strip the trailing whitespace
        row.rstrip()

        # Parse the elements
        (d, t, depth, status) = row.split(maxsplit=3)

        # Use date and time strings to make numpy datetime object
        ping_time[idx] = np.datetime64(datetime.strptime(d + t, "%Y%m%d%H%M%S%f"))

        # Convert depth and status to floats
        depth = convert_float(depth)
        if ignore_status:
            # Assume all points are valid
            status = 3
        else:
            status = convert_float(status)

        # Assign the depth value based on the status. For our purposes, any status
        # less than 3 in an .evl file will be considered "bad" and assigned NaN. Also,
        # .evl files have a special value (-10000.99000 ) used to indicate an invalid
        # sounder detected bottom vertex and these will also be assigned NaN.
        if status < 3 or depth < -10000.0:
            # This is an invalid vertex
            depth_data[idx] = np.nan
        else:
            depth_data[idx] = depth

    # Create the line object to return
    ev_line = line(ping_time=ping_time, data=depth_data, name=name, **kwargs)

    return ev_line

def read_xyz(xyz_filename, name='xyz_line', as_range=False, calibration=None,
        **kwargs):
    '''read_xyz will read a Simrad EK80 .xyz file and return a line object
    containing the bottom detection data.

    xyz_filename (string): The full path to the .xyz file to read
    name (string): name or label for the line.

    calibration (EK80.ek80_calibration): Set to an instance of EK80.ek80_calibration
                containing the calibration parameters
                you used when transforming to Sv/sv. The shound speed value in
                the calibration object will be used to shift the bottom detections
                if the sound speed used during data recording is different than
                the sound speed specified in this object.

    color: color is a list which defines the color of the line.
    linestyle: linestyle is a string that defines the style of the line.
    thickness: thickness is a float the defines the width of the line.

    '''

    import os
    from datetime import datetime

    def convert_float(val):
        try:
            val = float(val)
        except:
            val = np.nan
        return val

    def dms_to_dec(dms):
        try:
            parts = dms.split('.')
            val = float(parts[0]) + float(parts[1]) / 60.
        except:
            val = np.nan
        return val

    # Normalize filename and read the file
    xyz_filename = os.path.normpath(xyz_filename)
    with open(xyz_filename, 'r') as infile:
        xyz_data = infile.readlines()

    #  determine the number of line vertices
    n_points = len(xyz_data)

    # Simrad .xyz files contain Lat, Lon, depth, date, time, and transducer draft
    depth_data = np.empty((n_points), dtype=np.float32)
    lat_data = np.empty((n_points), dtype=np.float32)
    lon_data = np.empty((n_points), dtype=np.float32)
    draft_data = np.empty((n_points), dtype=np.float32)
    ping_time = np.empty((n_points), dtype='datetime64[ms]')

    # Loop thru the rows of data, parsing each line
    for idx, row in enumerate(xyz_data):
        # Parse the elements
        (lat, lon, depth, date, time, draft) = row.split()

        # Convert the time elements to datetime64
        ping_time[idx] = np.datetime64(datetime.strptime(date + time, "%d%m%Y%H%M%S.%f"))

        # Convert depth and draft to floats
        draft_data[idx] = convert_float(draft)
        if as_range:
            depth_data[idx] = convert_float(depth) - draft_data[idx]
        else:
            depth_data[idx] = convert_float(depth)

        # Convert lat, lon from DMS to decimal degrees
        lat_data[idx] = dms_to_dec(lat)
        lon_data[idx] = dms_to_dec(lon)

    # Create the line object to return
    xyz_line = line(ping_time=ping_time, data=depth_data, name=name, **kwargs)

    # Add the additional data attributes
    xyz_line.add_data_attribute('latitude', lat_data)
    xyz_line.add_data_attribute('longitude', lon_data)
    xyz_line.add_data_attribute('transducer_draft', draft_data)

    return xyz_line


import csv
import sys
import matplotlib
matplotlib.use("TkAgg")
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
from matplotlib.figure import Figure
from matplotlib import style
from tkinter import *
from tkinter import ttk
from tkinter.filedialog import askopenfilename
from tkinter import simpledialog
from tkinter import messagebox
from cycler import cycler
from PyQt5 import QtGui
import numpy as np

style.use('ggplot')
LARGE_FONT= ("Verdana", 12)

class Plotter(Tk):
    def __init__(self, *args, **kwargs):
        Tk.__init__(self, *args, **kwargs)
        Tk.iconbitmap(self, default="sincos.ico")
        Tk.wm_title(self, "Plotter")
        #Data holders
        self.plot_arrays_dict = {}
        self.plot_col_dict = {}
        self.Filename = ''
        self.Y_index_list = []
        self.Y2_index_list = []
        self.X_index = ''

        container = ttk.Frame(self)
        container.pack(side="top", fill="both", expand = True)
        container.grid_rowconfigure(0, weight=1)
        container.grid_columnconfigure(0, weight=1)
        #Frames hold the app pages
        self.frames = {}
        self.frames[StartPage] = StartPage(container, self)
        self.frames[StartPage].grid(row=0, column=0, sticky='nsew')
        #all page must be added to following tuple
        '''
        for F in (StartPage, WhatToPlot):
            frame=F(container, self)
            self.frames[F] = frame
            #defines the grid
            frame.grid(row=0, column=0, sticky="nsew")
        '''
        self.show_frame(StartPage)

    def show_frame(self, cont):
        #Raises the chosen page to the top.
        frame = self.frames[cont]
        frame.tkraise()

class StartPage(ttk.Frame):
    def __init__(self, parent, controller):
        Frame.__init__(self,parent)
        self.axis_1_colours = ['r','g','b','y','navy']    #sets different colour cyclers for each axis to avoid confusion
        self.axis_2_colours = ['y','m','k','gray','greenyellow']
        self.graph_set = False
        self.bind("<<ShowGraphPage>>", lambda event:self.on_show_graph_page(controller))
        self.Graph= Figure(figsize=(1,1), dpi=100)
        self.canvas = FigureCanvasTkAgg(self.Graph, self)
        self.canvas.get_tk_widget().pack(side="bottom", fill="both", expand = True, pady=5)
        self.toolbar = NavigationToolbar2TkAgg(self.canvas, self)
        self.toolbar.update()
        self.canvas._tkcanvas.pack(side="top", fill="both", expand = True)
        self.sp1_1 = self.Graph.add_subplot(111)
        self.sp1_1.set_prop_cycle(cycler('color', self.axis_1_colours))

        load_file_butt = ttk.Button(self, text="Choose data file", command=lambda: self.load_file(controller))
        load_file_butt.pack()

        configure_plot_butt = ttk.Button(self, text='Configure axis', command=lambda: self.config_plot(controller))
        configure_plot_butt.pack()

        plot_button = ttk.Button(self, text='Plot it', command=lambda: self.plot_it(controller))
        plot_button.pack()

        clr_plot_butt = ttk.Button(self, text='Clear plot', command=self.clear_plt)
        clr_plot_butt.pack()

    def load_file(self, controller):
        controller.Filename = askopenfilename(initialdir="Z:\\", title="Choose a file")
        extract_data(controller)

    def config_plot(self, controller):
        if controller.Filename == '':
            messagebox.showerror('No file loaded!')
        elif controller.data_titles == []:
            messagebox.showerror('Unrecognised file format')
        else:
            WhatToPlot(controller)

    def plot_it(self, controller):
        if controller.Filename == '':
            messagebox.showerror('No file loaded!')
        elif controller.X_index == '':
            messagebox.showerror('No X axis data')
        elif controller.Y_index_list == []:
            messagebox.showerror('No Y axis data')
        else:
            for i in controller.Y_index_list:
                self.sp1_1.plot(controller.plot_arrays_dict[controller.X_index], controller.plot_arrays_dict[i], 'o')
            self.sp1_1.set_xlabel(controller.data_titles[controller.X_index])
            self.sp1_1.set_ylabel(controller.data_titles[controller.Y_index_list[0]]) #This should grab the first title that was plotted and set it
            if controller.Y2_index_list != []:
                self.sp1_2=self.sp1_1.twinx()
                self.sp1_2.set_prop_cycle(cycler('color', self.axis_2_colours))
                for i in controller.Y2_index_list:
                    self.sp1_2.plot(controller.plot_arrays_dict[controller.X_index], controller.plot_arrays_dict[i], 'o')
                self.sp1_2.set_ylabel(controller.data_titles[controller.Y2_index_list[0]])
            self.canvas.draw()

    def clear_plt(self):
    	self.Graph.clear()
    	self.sp1_1 = self.Graph.add_subplot(111)
    	self.sp1_1.set_prop_cycle(cycler('color', self.axis_1_colours))
    	self.canvas.draw()

class WhatToPlot(Toplevel):
    def __init__(self, controller):
        Toplevel.__init__(self)
        self.geometry("500x300")
        self.title("What do you want to plot?")
        XLabel = ttk.Label(self, text='X-axis', font=LARGE_FONT)
        XLabel.grid(row=1,column=1, padx=20)
        YLabel = ttk.Label(self, text='Y-axis ?', font=LARGE_FONT)
        YLabel.grid(row=1,column=2, padx=20)
        Y2Label = ttk.Label(self, text='Y-axis 2', font=LARGE_FONT)
        Y2Label.grid(row=1,column=3, padx=20)

        self.X=ttk.Combobox(self, values=controller.data_titles)
        self.X.grid(row=2, column=1)

        Y_1=ttk.Combobox(self, values=controller.data_titles)
        Y_1.grid(row=2,column=2)
        Y_2=ttk.Combobox(self, values=controller.data_titles)
        Y_2.grid(row=3,column=2)
        Y_3=ttk.Combobox(self, values=controller.data_titles)
        Y_3.grid(row=4,column=2)
        Y_4=ttk.Combobox(self, values=controller.data_titles)
        Y_4.grid(row=5,column=2)
        Y_5=ttk.Combobox(self, values=controller.data_titles)
        Y_5.grid(row=6,column=2)

        Y2_1=ttk.Combobox(self, values=controller.data_titles)
        Y2_1.grid(row=2,column=3)
        Y2_2=ttk.Combobox(self, values=controller.data_titles)
        Y2_2.grid(row=3,column=3)
        Y2_3=ttk.Combobox(self, values=controller.data_titles)
        Y2_3.grid(row=4,column=3)
        Y2_4=ttk.Combobox(self, values=controller.data_titles)
        Y2_4.grid(row=5,column=3)
        Y2_5=ttk.Combobox(self, values=controller.data_titles)
        Y2_5.grid(row=6,column=3)

        self.Y_obj_list = [Y_1, Y_2, Y_3, Y_4, Y_5]
        self.Y2_obj_list = [Y2_1, Y2_2, Y2_3, Y2_4, Y2_5]

        conf_butt = ttk.Button(self, text="Confirm and close", command=lambda: self.confirm_and_close(controller))
        conf_butt.grid(row=7, column=2, columnspan=2)

    def confirm_and_close(self, controller):
        controller.Y_index_list = self.check_vals(self.Y_obj_list, controller.data_titles)
        controller.Y2_index_list = self.check_vals(self.Y2_obj_list, controller.data_titles)
        controller.X_index = controller.data_titles.index(self.X.get())
        self.destroy()

    def check_vals(self, obj_list, data_titles):
        vals_list = []
        index_to_plot_list = []
        for o in obj_list:
            if o.get() == '':
                pass
            else:
                vals_list.append(o.get())
        for val in vals_list:
            ind = data_titles.index(val)
            index_to_plot_list.append(ind)
        return index_to_plot_list

def extract_data(controller):
    controller.plot_arrays_dict={}
    with open(controller.Filename) as csv_file:
        read_csv = csv.reader(csv_file, delimiter=',')
        for index, row in enumerate(read_csv):
            if len(row)>0:
                if index==0:
                    controller.data_titles = row
                    number_cols = len(row)
                    for i in range(number_cols):
                        controller.plot_col_dict[i]=[]
                else:
                    for i in range(number_cols):
                        controller.plot_col_dict[i].append(row[i])
    for i in range(number_cols):
        controller.plot_arrays_dict[i]=np.asarray(controller.plot_col_dict[i], dtype='float')

app = Plotter()
app.geometry("1000x800")
app.mainloop()


#     Data Types
#...1. String
#print("Hello"[4]) #--this is called subscript
#print("Hello " + "345")
#..2. Integer
#print(123 + 456)
#..3. Float
#3125.6
#..4. Boolean
#True
#False

# Data type Excercise

# street_name = "Abbey Road"
# print(street_name[4] + street_name[7])  # the result would be yo

#Type Error, checking and type conversion

# num_char = len(input("What is your name? "))
# new_num_char = str(num_char)
# print("Your name has " + new_num_char + " characters.") # corrected by using type function
# print("Your name has " + num_chart + " characters.") # data type Error

#Excercise
# Write a program that adds the digits in a 2 digit number. e.g. if the input was 35, then the output should be 3 + 5 = 8
# two_digit_number = input("Type a two digit number: ")
# first_digit = two_digit_number[0]
# second_digit = two_digit_number[1]
# result = int(first_digit) + int(second_digit)
# print(result)

#Math Operation
# operations are based on PEMDASLR - parentheses, exponents, multiplicattion, division, addition, subtraction
# print (3 * 3 + 3 / 3 - 3) # example 1
# print(3 * (3 + 3) / 3 -3 ) # example 2

# BMI Calculator
# The BMI is calculated by dividing a person's weight (in kg) by the square of their height (in m):
# height = input("enter your height in m: ")
# weight = input("enter your weight in kg: ")
# bmi = int(weight) / float(height) ** 2
# bmi_as_int = int(bmi)
# print(bmi_as_int)

#.... Number manipulation
     # the round() filter
# print(round(8 / 3))
# print(round(8 / 3, 2)) # rounding to two digits
# print(8 // 3)

# result = 4 / 2
# result /= 2 # we can use +=, -+, */ or /=
# print(result)

# F string helps to avoid converting data types and writing long codes.
# score = 0
# height = 1.8
# isWinning = True
# print(f"your score is {score}, your height is {height}, you are winning is {isWinning}")

# name = input("What is your name? ") # example 1
# print(f"Hello, {name}")
# name = input("What is your name? ") # example 2
# print("Hello, " + name)

# age = 12  # example 3
# print(f"You are {age} years old")

# Excercise: Your life in weeks
# example out put you have 12410 days, 1768 weeks and 408 months left.

# age = input("what is your current age? ")

# age_as_int = int(age)
# years_remaining = 90 - age_as_int
# days_remaining = years_remaining * 365
# weeks_remaining = years_remaining * 52
# months_remaining = years_remaining * 12

# message = f"You have {days_remaining} days,or {weeks_remaining} weeks,or {months_remaining} months or {years_remaining} years left."
# print(message)

# Excercise Tip Calculator

# print("Welcome to the tip calculator")
# # print("Welcome to the Payroll calculator!")
# bill = float(input("What was the total bill? $"))

# tip = int(input("How much tip would you like to give? 10, 12, or 15? "))
# people = int(input("How many people to split the bill?"))
# tip_as_percent = tip / 100
# total_tip_amount = bill * tip_as_percent
# total_bill = bill + total_tip_amount
# bill_per_person = total_bill / people
# final_amount = round(bill_per_person, 2)
# print(f"Each person should pay: ${final_amount}")

# Pyroll calculator project
n = 1
while n < 5:

    print("Welcome to the payroll calculator!")
    #Employee Name
    employee = input("Employee Name: ")
    # Hours Worked
    hours_worked = float(input("Hour Worked: "))
    extra_hours_worked = float(input("Extra Hour Worked: "))
    # Payment Rates
    pay_rate = float(input("Pay Rate $: "))
    extra_hours_rate = 1.5 * pay_rate
    #Payements
    reg_pay = hours_worked * pay_rate
    ext_pay = extra_hours_worked * extra_hours_rate
    gross_pay = reg_pay + ext_pay
    # Tax Rates
    fed_tax_rate = 0.10
    state_rate = 0.03
    fica_rate = 0.07

    #Total tax
    fed_tax = round(pay_rate * fed_tax_rate, 2)
    state_tax = round(pay_rate * state_rate, 2)
    fica_tax = round(pay_rate * fica_rate, 2)
    total_tax = round(fed_tax + state_tax + fica_tax, 2)
    net_pay = round(gross_pay - total_tax, 2)

    print(f"Your gross payment is {gross_pay} dollars.")
    print(f"Your Total Federal tax is {pay_rate * fed_tax_rate} dollar.")
    print(f"Your State tax is {pay_rate * state_rate} dollar.")
    print(f"Your FICA payment is {fica_tax} dollar.")
    # print(f"Your FICA payment is {pay_rate * fica_rate} dollar.")
    print(f"Your total tax payed is {total_tax} dollar.")
    print(f"Your net payment is {net_pay} dollar.")
    print(f"Thank you dear {employee}!")

    n += 1


#!/usr/bin/env python3
# -*- coding: utf-8 -#-

import binascii

chars = ['A', 'å']
encodings = ['utf-8', 'latin1', 'utf-16-le', 'utf-16-be']

for ch in chars:
    for enc in encodings:
        b = ch.encode(enc)
        print(ch, 'in', enc, '-->', binascii.hexlify(b))


import pandas as pd
import re
import time
import random
import os
import string
import csv
import json

def Counter():
    s = 0
    def counter():
        nonlocal s
        s = s + 1
        return s
    return counter

def get_str_time():
    return time.strftime('%Y%m%d%H%M%S')

def str2list(s,delimite=r'[\t| ]'):
    return re.split(delimite,s)

def str2dict(s,linebreak='\n',delimite=r'[\t| ]',):
    result={}
    for line in re.split(linebreak,s):
        temp=[]
        for item in re.split(delimite,line):
            if item=='':
                continue
            else:
                temp.append(item)
        if len(temp)==2:
            result[temp[0]]=temp[1]
        elif len(temp)==1:
            result[temp[0]] = ''
        elif len(temp)==0:
            continue
        else:
            print(line,'无法解析，请检查')
    return result

def get_random_str(length=8):
    ran_str = ''.join(random.sample(string.ascii_letters + string.digits, length))
    return ran_str

def get_files_in_subdir(base_dir='.',ext=[]):

    print('注意：\r\n'
          '文件类型必须传入数组,如[jpg],[txt,rar]，为空不筛选'
          )
    base_dir=base_dir
    base_dir=os.path.abspath(base_dir)
    ext=['.'+str.lower(x) for x in ext ]
    result=[]

    try:
        if os.path.exists(base_dir) and os.path.isdir(base_dir):
            for sub_dir in os.listdir(base_dir):
                if os.path.isdir(os.path.join(base_dir,sub_dir)):
                    for file in os.listdir(os.path.join(base_dir,sub_dir)):
                        if os.path.isfile(os.path.join(base_dir,sub_dir,file)):
                            filename,extname=os.path.splitext(file)
                            if ext:
                                if str.lower(extname) in ext:
                                    result.append(tuple([base_dir,sub_dir,filename,extname]))
                            else:
                                result.append(tuple([base_dir, sub_dir, filename, extname]))
    except Exception as e:
        print(e)
    for item in result:
        print(item)
    return result

def get_files_in_dir(basepath,ext=[],debug=False):

    basepath=os.path.abspath(basepath)

    ext = [str.lower(x) for x in ext]

    try:
        if os.path.exists(basepath) and os.path.isdir(basepath):
            result=[os.path.join(basepath,x) for x in os.listdir(basepath)]
            files=[[os.path.split(x)[0],*os.path.splitext(os.path.basename(x))] for x in result if os.path.isfile(x)]
            if len(ext)==0:
                print('无类型筛选,返回所有文件')
                if debug:print(files)
                return files

            if len(ext)>0 :
                print('返回{}类型的文件'.format(ext))
                files=[[os.path.split(x)[0],*os.path.splitext(os.path.basename(x))] for x in result if str.lower(re.split(r'\.',x)[-1]) in ext]
                if debug:print(files)
                return files
        else:
            return False
    except Exception as e:
        print('发生错误{}'.format(e))

def createDataFrame(columns):
    return pd.DataFrame(columns=columns)

def appendRecoder2DataFrame(data,record=dict):#append方式，单条添加，类型为字典
    result= data.append(record, ignore_index=True)
    return result

def concatRecoder2DataFrame(data,records=[],axis=0):#concat方式，可一次添加多条，多列，传入存有数据字典的列表
    records=pd.DataFrame(records)
    result= pd.concat([data,records],axis=axis,ignore_index=True)
    return result

def saveAsExcel(f,dataFrame):
    if isinstance(dataFrame,pd.DataFrame):
        dataFrame.to_excel(f,encoding='utf_8_sig',index=False)

def saveAsTxt(fpath,content,linehead='\t',linebreak='\r\n',codec='utf-8',mode:str='w'):
    if isinstance(content,str):
        with open(fpath,mode,encoding=codec) as f:
            f.write(content)
        return True

    if isinstance(content,list):
        with open(fpath, mode, encoding=codec) as f:
            for line in content:
                f.write(linehead+line+linebreak)
            return True

    if isinstance(content,dict):
        with open(fpath, mode, encoding=codec) as f:
            for key in content.keys():
                item=content[key]
                f.write(key+linebreak)
                if isinstance(item,str):
                    f.write(linehead+item+linebreak)
                if isinstance(item,list):
                    for line in item:
                        f.write(linehead+line+linebreak)
        return True

def readTxt(file_path,codec='utf-8',mode='r'):
    content=None
    with open(file_path,mode=mode,encoding=codec) as f:
        content=f.read()
    return content

def write_csv(filepath, line: list):
    with open(filepath, 'a', newline='', encoding='utf-8-sig') as f:
        csv_file = csv.writer(f, delimiter=',')
        csv_file.writerow(line)

def write_csv_with_name(filepath, line: list, columns: list):
    if os.path.exists(filepath):
        with open(filepath, 'a', newline='', encoding='utf-8-sig') as f:
            csv_file = csv.writer(f, delimiter=',')
            csv_file.writerow(line)
    else:
        with open(filepath, 'a', newline='', encoding='utf-8-sig') as  f:
            csv_file = csv.writer(f, delimiter=',')
            csv_file.writerow(columns)
            csv_file.writerow(line)

def save_json(filepath,data):
    with open(filepath,'w',encoding='utf-8') as f:
        f.write(json.dumps(data,indent=2,ensure_ascii=False))

def parse(filepath):
    pass

if __name__ == '__main__':

    output_file='output.csv'
    sourcedir='source'

    for basedir,name,ext in get_files_in_dir(sourcedir):
        filepath=os.path.join(basedir,name+ext)
        parse(filepath=filepath)
        m=parse(filepath)

my_model/读取目录文件并处理的模板.py

读取目录文件并处理的模板.py


import os
import re

class dictionary:
    def __init__(self):
        self.pre_s = []
        with open("pre_changed.txt", "r") as file:
            for line in file:
                line = line.replace("\n", "")
                self.pre_s.append(line)
        self.dict=[]
        with open("common_words.txt","r") as file:
            for i in file:
                self.dict.append(i.replace("\n","").replace("\r",""))

    def __getitem__(self, word,max_features):
        word=self.find_root(word)
        for i in range(len(self.dict)):
            if self.dict[i]==word:
                result=int(self.dict[i-1])
                if result<max_features-1:
                    return int(self.dict[i-1])
                else:
                    return max_features-2
        return max_features-1

    def find_root(self,word):
        word=re.sub(r"[,.!?_()#@$%&*+=/1234567890«»:;]", "", word).replace("А","а").replace("Б","б").replace("В","в").replace("Г","г").replace("Д","д").replace("Е","е").replace("Ж","ж").replace("З","з").replace("И","и").replace("Й","й").replace("К","к").replace("Л","л").replace("М","м").replace("Н","н").replace("О","о").replace("П","п").replace("Р","р").replace("С","с").replace("Т","т").replace("У","у").replace("Ф","ф").replace("Х","х").replace("Ц","ц").replace("Ч","ч").replace("Щ","щ").replace("Ш","ш").replace("Ы","ы").replace("Э","э").replace("Ю","ю").replace("Я","я")

        for i in self.pre_s:
                if (word.startswith(i)):
                    #print(i)#, end="")
                    suspected_root=word[len(i):].zfill(4)[:4].replace("0","")
                    if suspected_root!="0000":
                        return(suspected_root)
        return word.zfill(4)[:4].replace("0","")

    def convert_sentence(self,sentence,max_features):
        words_vector=[]
        for i in sentence.replace("\n"," ").strip().split(" "):
            words_vector.append(self.__getitem__(i,max_features))
        return words_vector

Dict=dictionary()
#print(Dict.convert_sentence("итак, будем веселиться, пока мы молоды! После веселой молодости, после горестной старости"))

#db=mysql_connect.connect()
sum=0
arr=[]

def extract_train_from_file(mode,dir_first,dir_second,max_features):
    lengts_poems=[]
    result_train=[]
    dir1="poem_base/"+dir_first+"/"+mode+dir_second+"/texts/"
    if mode=="test/":
       num_files_1 = (len(os.listdir(dir1)))
       print(num_files_1)
       file_current=1
    else:
        num_files_1 = (len(os.listdir(dir1)))-1
        print(num_files_1)
        file_current = 0
    while file_current <=num_files_1:
        length_poem=0
        with open("poem_base/"+dir_first+"/"+mode+dir_second+"/texts/" + str(file_current) + ".txt", "r",encoding="utf-8") as file:
            four_rows = ""
            num_rows = 0
            for i in file:
                print(i)
                num_rows += 1
                four_rows += i
                if num_rows == 4:
                    length_poem+=1
                    result_train.append(Dict.convert_sentence(four_rows, max_features))
                    four_rows = ""
                    num_rows = 0
        file_current+=1
        if mode=="test/":
            with open("poem_base/"+dir_first+"/length_poems.txt","a+") as file:
                file.write(str(length_poem)+",")
        else:
            with open("poem_base/"+dir_first+"/tag_length_poems.txt","a+") as file:
                file.write(str(length_poem)+",")

    print(result_train)
    return result_train

def write_result_to_files(mode_write,mode,dir_first,dir_second,max_features):
    vector_result=extract_train_from_file(mode,dir_first,dir_second,max_features)
    for i in range(len(vector_result)):
        with open ("poem_base/"+dir_first+"/"+mode_write+dir_second+"/"+str(i)+".txt","w") as file:
            for j in vector_result[i]:
                file.write(str(j)+"\n")


""" File name:   truth_tables.py
    Author:      Rui Qiu
    Date:        23 Feb, 2018
    Description: This file defines a number of functions which implement
                 Boolean expressions.

                 It also defines a function to generate and print truth tables
                 using these functions.

                 It should be implemented for Exercise 2 of Assignment 0.

                 See the assignment notes for a description of its contents.
"""

def boolean_fn1(a, b, c):
    """ Return the truth value of (a ∨ b) → (-a ∧ -b)

    :param a: Boolean.
    :param b: Boolean.
    :param c: Boolean.
    :return: Truth value of (a ∨ b) → (-a ∧ -b).
    """

    left = a or b
    right = not a and not b
    return (not left) or right

def boolean_fn2(a, b, c):
    """ Return the truth value of (a ∧ b) ∨ (-a ∧ -b)

    :param a: Boolean.
    :param b: Boolean.
    :param c: Boolean.
    :return: Truth value of (a ∧ b) ∨ (-a ∧ -b).
    """

    return (a and b) or (not a and (not b))

def boolean_fn3(a, b, c):
    """ Return the truth value of ((c → a) ∧ (a ∧ -b)) ∨ (-a ∧ b)

    :param a: Boolean.
    :param b: Boolean.
    :param c: Boolean.
    :return: Truth value of ((c → a) ∧ (a ∧ -b)) ∨ (-a ∧ b)
    """

    first = not c or a
    return first and (a and not b) or (not a and b)

def draw_truth_table(boolean_fn):
    """ This function prints a truth table for the given boolean function.
        It is assumed that the supplied function has three arguments.

        ((bool, bool, bool) -> bool) -> None

        If your function is working correctly, your console output should look
        like this:

        >>> from truth_tables import *
        >>> draw_truth_table(boolean_fn1)
        a     b     c     res
        -----------------------
        True  True  True  False
        True  True  False False
        True  False True  False
        True  False False False
        False True  True  False
        False True  False False
        False False True  True
        False False False True

    :param boolean_fn: A pre-defined function which takes 3 boolean variables
    as input.
    :return: A string displayed as a truth table which includes all the
    combinations of
    input values, and the corresponding truth values of that Boolean formula.
    """

    combo = [(x, y, z) for x in [True, False] for y in [True, False]
             for z in [True, False]]

    print(
        "a     b     c     res\n"
        "-----------------------\n"
        "True  True  True  "+str(boolean_fn(
            combo[0][0], combo[0][1], combo[0][2]))+"\n"
        "True  True  False "+str(boolean_fn(
            combo[1][0], combo[1][1], combo[1][2]))+"\n"
        "True  False True  "+str(boolean_fn(
            combo[2][0], combo[2][1], combo[2][2]))+"\n"
        "True  False False "+str(boolean_fn(
            combo[3][0], combo[3][1], combo[3][2]))+"\n"
        "False True  True  "+str(boolean_fn(
            combo[4][0], combo[4][1], combo[4][2]))+"\n"
        "False True  False "+str(boolean_fn(
            combo[5][0], combo[5][1], combo[5][2]))+"\n"
        "False False True  "+str(boolean_fn(
            combo[6][0], combo[6][1], combo[6][2]))+"\n"
        "False False False "+str(boolean_fn(
            combo[7][0], combo[7][1], combo[7][2])))


spaceIndices = []
WordBank = []
spam = "As the stock of Tesla goes up, the cost continues to rise with it. This is very intimidating to other electric car companies."

print(spam)
newSpam = spam.strip()
print(newSpam)

print(newSpam.find(" "));

print(newSpam.rfind(" "));

letters = newSpam
i = 0
for let in letters:
  print(str(i) + " - " + let)
  if(let == (" ")):
    spaceIndices.append(i)
  i = i+1

print(spaceIndices)

for let in letters:
  print(str(i))


"""Implementation of the VQ-VAE [1] and VQ-VAE-2 [2] models.

The Vector Quantized Variational Autoencoder (VQ-VAE) extends the vanilla
Variational Autoencoder by mapping the input to a discrete (instead of
continuous) latent space. The discretization is performed using vector
quantization where continuous outputs from an encoder are mapped to the nearest
vectors in a learned codebook. The main intuition behind using discretized
latents is that the underlying domain we are trying to model is usually
discrete (e.g. words in a sentence, objects in an image, etc).

VQ-VAE-2 further extends the original VQ-VAE by encoding the input to a
hierarchy of latent spaces. The top latent space is responsible for encoding
high level semantic details with progressively lower level details being encoded
as down the hierarchy.

References (used throughout the code):
  [1]: https://arxiv.org/pdf/1711.00937.pdf
  [2]: https://arxiv.org/pdf/1906.00446.pdf
"""

import torch
from torch import nn

from pytorch_generative import nn as pg_nn

class ResidualBlock(nn.Module):
  """A simple residual block."""

  def __init__(self, n_channels, hidden_channels):
    """Initializes a new ResidualBlock instance.

    Args:
      n_channels: Number of input and output channels.
      hidden_channels: Number of hidden channels.
    """
    super().__init__()
    self._net = nn.Sequential(
        nn.ReLU(),
        nn.Conv2d(in_channels=n_channels, out_channels=hidden_channels,
                  kernel_size=3, padding=1),
        nn.ReLU(),
        nn.Conv2d(in_channels=hidden_channels, out_channels=n_channels,
                  kernel_size=1))

  def forward(self, x):
    return x + self._net(x)

class ResidualStack(nn.Module):
  """A stack of multiple ResidualBlocks."""

  def __init__(self, n_channels, hidden_channels, n_residual_blocks=1):
    """Initializes a new ResidualStack instance.

    Args:
      n_channels: Number of input and output channels.
      hidden_channels: Number of hidden channels.
      n_residual_blocks: Number of residual blocks in the stack.
    """
    super().__init__()
    self._net = nn.Sequential(*[
        ResidualBlock(n_channels, hidden_channels)
        for _ in range(n_residual_blocks)
    ] + [nn.ReLU()])

  def forward(self, x):
    return self._net(x)

class Encoder(nn.Module):
  """A feedforward encoder which downsamples its input."""

  def __init__(self,
               in_channels,
               out_channels,
               hidden_channels,
               n_residual_blocks,
               residual_channels,
               stride):
    """Initializes a new Encoder instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in non residual block hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      stride: Stride to use in the downsampling convolutions. Must be even.
    """
    super().__init__()
    assert stride % 2 == 0, '"stride" must be even.'

    net = []
    for i in range(stride//2):
      first, last = 0, stride // 2 - 1
      in_c = in_channels if i == first else hidden_channels//2
      out_c = hidden_channels //2 if i < last else hidden_channels
      net.append(
          nn.Conv2d(in_channels=in_c, out_channels=out_c, kernel_size=4,
                    stride=2, padding=1))
      net.append(nn.ReLU())
    net.append(
        nn.Conv2d(in_channels=hidden_channels, out_channels=hidden_channels,
                  kernel_size=3, padding=1))
    net.append(
        ResidualStack(n_channels=out_channels,
                      hidden_channels=residual_channels,
                      n_residual_blocks=n_residual_blocks))
    self._net = nn.Sequential(*net)

  def forward(self, x):
    return self._net(x)

class Decoder(nn.Module):
  """A feedforward encoder which upsamples its input."""

  def __init__(self,
               in_channels,
               out_channels,
               hidden_channels,
               n_residual_blocks,
               residual_channels,
               stride):
    """Initializes a new Decoder instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in (non residual block) hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      stride: Stride to use in the upsampling (i.e. transpose) convolutions.
        Must be even.
    """
    super().__init__()

    assert stride % 2 == 0, '"stride" must be even.'

    net = [
      nn.Conv2d(in_channels=in_channels, out_channels=hidden_channels,
                kernel_size=3, padding=1),
      ResidualStack(n_channels=hidden_channels,
                    hidden_channels=residual_channels,
                    n_residual_blocks=n_residual_blocks)
    ]
    for i in range(stride//2):
      first, last = 0, stride//2 - 1
      in_c = hidden_channels if i == first else hidden_channels//2
      out_c = hidden_channels//2 if i < last else out_channels
      net.append(
          nn.ConvTranspose2d(in_channels=in_c, out_channels=out_c,
                             kernel_size=4, stride=2, padding=1))
      if i < last:
        net.append(nn.ReLU())
    self._net = nn.Sequential(*net)

  def forward(self, x):
    return self._net(x)

class Quantizer(nn.Module):
  """Wraps a VectorQuantizer to handle input with arbitrary channels."""

  def __init__(self, in_channels, n_embeddings, embedding_dim):
    """Initializes a new Quantizer instance.

    Args:
      in_channels: Number of input channels.
      n_embeddings: Number of VectorQuantizer embeddings.
      embedding_dim: VectorQuantizer embedding dimension.
    """
    super().__init__()
    self._net = nn.Sequential(
        nn.Conv2d(in_channels=in_channels, out_channels=embedding_dim,
                  kernel_size=1),
        pg_nn.VectorQuantizer(n_embeddings, embedding_dim))

  def forward(self, x):
    return self._net(x)

class VQVAE(nn.Module):
  """The Vector Quantized Variational Autoencoder (VQ-VAE) model."""

  def __init__(self,
               in_channels=1,
               out_channels=1,
               hidden_channels=128,
               n_residual_blocks=2,
               residual_channels=32,
               n_embeddings=128,
               embedding_dim=16):
    """Initializes a new VQVAE instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in (non residual block) hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      n_embeddings: Number of VectorQuantizer embeddings.
      embedding_dim: Dimension of the VectorQuantizer embeddings.
    """
    super().__init__()
    self._encoder = Encoder(in_channels=in_channels,
                            out_channels=hidden_channels,
                            hidden_channels=hidden_channels,
                            n_residual_blocks=n_residual_blocks,
                            residual_channels=residual_channels,
                            stride=4)
    self._quantizer = Quantizer(in_channels=hidden_channels,
                                n_embeddings=n_embeddings,
                                embedding_dim=embedding_dim)
    self._decoder = Decoder(in_channels=embedding_dim,
                            out_channels=out_channels,
                            hidden_channels=hidden_channels,
                            n_residual_blocks=n_residual_blocks,
                            residual_channels=residual_channels,
                            stride=4)

  def forward(self, x):
    x = self._encoder(x)
    quantized, quantization_loss = self._quantizer(x)
    return self._decoder(quantized), quantization_loss

class VQVAE2(nn.Module):
  """The VQ-VAE-2 model with a latent hierarchy of depth 2."""

  def __init__(self,
               in_channels=1,
               out_channels=1,
               hidden_channels=128,
               n_residual_blocks=2,
               residual_channels=32,
               n_embeddings=128,
               embedding_dim=16):
    """Initializes a new VQVAE2 instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in (non residual block) hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      n_embeddings: Number of VectorQuantizer embeddings.
      embedding_dim: Dimension of the VectorQuantizer embeddings.
    """
    super().__init__()

    self._encoder_b = Encoder(in_channels=in_channels,
                              out_channels=hidden_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)
    self._encoder_t = Encoder(in_channels=hidden_channels,
                              out_channels=hidden_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)
    self._quantizer_t = Quantizer(in_channels=hidden_channels,
                                  n_embeddings=n_embeddings,
                                  embedding_dim=embedding_dim)
    self._quantizer_b = Quantizer(in_channels=hidden_channels,
                                  n_embeddings=n_embeddings,
                                  embedding_dim=embedding_dim)
    self._decoder_t = Decoder(in_channels=embedding_dim,
                              out_channels=hidden_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)
    self._conv = nn.Conv2d(in_channels=in_channels, out_channels=embedding_dim,
                          kernel_size=1)
    self._decoder_b = Decoder(in_channels=2*embedding_dim,
                              out_channels=out_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)

  def forward(self, x):
    encoded_b = self._encoder_b(x)
    encoded_t = self._encoder_t(encoded_b)

    quantized_t, vq_loss_t = self._quantizer_t(encoded_t)
    quantized_b, vq_loss_b = self._quantizer_b(encoded_b)

    decoded_t = self._decoder_t(quantized_t)
    xhat = self._decoder_b(
        torch.cat((self._conv(decoded_t), quantized_b), dim=1))
    return xhat, .5 * (vq_loss_b + vq_loss_t) + F.mse_loss(decoded_t, encoded_b)


import argparse
import asyncio
import logging.config
import os
import sys
import typing
from collections import namedtuple

import aiohttp
import lxml.html
from yarl import URL

from . import storage
from .book import Book
from .notifier import Notifier

class Watcher:
    def __init__(self, notifier: Notifier, logger: logging.Logger = logging.getLogger('Watcher')):
        self.notifier = notifier
        self.logger = logger

    async def get_shelf_books(self, session: aiohttp.ClientSession, shelf: str) -> typing.Dict[str, 'Book']:
        async def create_book_from_book_bom(book_bom: lxml.html.Element) -> Book:
            title = ''.join(book_bom.xpath('td[2]/a/text()'))
            author = ''.join(book_bom.xpath('td[3]/text()'))
            publisher = ''.join(book_bom.xpath('td[4]/text()'))
            publish_date = ''.join(book_bom.xpath('td[5]/text()'))
            call_no = ''.join(book_bom.xpath('td[6]/text()'))
            marc_no_url = URL(''.join(book_bom.xpath('td[2]/a/@href')))
            marc_no = marc_no_url.query.get('marc_no')
            state = await self.get_book_state(session, marc_no)
            return Book(title, author, publisher, publish_date, call_no, marc_no, state)

        url = URL('http://210.32.33.91:8080/opac/show_user_shelf.php').with_query(classid=shelf)
        async with session.get(url) as response:
            try:
                html = await response.text()
            except Exception as e:
                self.logger.debug('Unable to access shelf {} page'.format(shelf), exc_info=True)
                raise e
            else:
                self.logger.debug('Access {} shelf page success'.format(shelf))

                bom = lxml.html.fromstring(html)
                book_bom_list = bom.xpath('//*[@id="container"]/table/tr')[1:]

                _loop = asyncio.get_event_loop()
                book_tasks = [_loop.create_task(create_book_from_book_bom(book_bom)) for book_bom in book_bom_list]
                finished, unfinished = await asyncio.wait(book_tasks)

                book_dict = {}  # type: typing.Dict[str, 'Book']
                for r in finished:
                    book = r.result()
                    if not isinstance(book, Book):
                        raise TypeError('{} is not a Book Object', )
                    book_dict[book.call_no] = book
                self.logger.debug('shelf book is {}'.format(book_dict))
                return book_dict

    async def get_book_state(self, session: aiohttp.ClientSession, marc_no: str):
        url = URL('http://210.32.33.91:8080/opac/ajax_item.php').with_query(marc_no=marc_no)
        async with session.get(url) as response:
            try:
                html = await response.text()
            except Exception as e:
                self.logger.warning('Unable to access ajax-page of {}'.format(marc_no), exc_info=True)
                raise e
            else:
                self.logger.debug('Access {} ajax-page success'.format(marc_no))
                dom = lxml.html.fromstring(html)
                state_list = dom.xpath('//*[@id="item"]/tr/td[5]/font/text() | //*[@id="item"]/tr/td[5]/text()')
                state = '可借' in state_list
                self.logger.debug('Parse {} result is {} {}'.format(marc_no, state, state_list))
                return state

    async def send_all_status_loop(self, loop_time=7 * 24 * 3600):
        remain_time = loop_time
        while remain_time >= 3600:
            remain_time = remain_time - 3600
            await asyncio.sleep(3600)
        store_books = storage.load()
        self.logger.debug('Send All Status')
        books = [Notifier.Notify(book, None) for book in store_books.values()]
        await self.notifier.send_all_status(books)
        asyncio.get_event_loop().create_task(self.send_all_status_loop())

    async def check_loop(self, shelf: str, loop_time: int = 3600):
        self.logger.info('-----------------------')
        self.logger.info('Watcher start')

        session = aiohttp.ClientSession()
        try:
            while True:
                try:
                    self.logger.debug('Checking...')

                    books = await self.get_shelf_books(session, shelf)  # type: typing.Dict[str, 'Book']
                    self.logger.debug('Get {} shelf books'.format(len(books)))

                    with storage.FLock():
                        store_books = storage.load()  # type: typing.Dict[str, 'Book']
                        self.logger.debug('Load {} stored books'.format(len(store_books)))

                        del_key_list = []
                        for call_no, store_book in store_books.items():
                            if store_book not in books.values():
                                self.logger.info('Stop track {}'.format(store_book))
                                # self.notifier.collect_notify(store_book, '停止追踪')
                                del_key_list.append(call_no)

                        for del_key in del_key_list:
                            del store_books[del_key]

                        for call_no, book in books.items():
                            if book not in store_books.values():
                                self.logger.info('Begin track {}'.format(book))
                                # self.notifier.collect_notify(book, '开始追踪')
                                store_books[call_no] = book
                            else:
                                if book.state != store_books.get(call_no).state:
                                    self.logger.info('Tracked book {} state change'.format(book))
                                    self.notifier.collect_notify(book, None)
                                    store_books[call_no] = book
                                else:
                                    self.logger.debug('Tracked book {} state not change'.format(book))

                        storage.dump(store_books)
                        self.logger.debug('Store {} books to file'.format(len(store_books.values())))

                    await self.notifier.send_notify()

                except TimeoutError:
                    self.logger.error('Watcher occurs error', exc_info=True)

                except Exception as e:
                    self.logger.error('Watcher occurs error', exc_info=True)
                    raise e

                finally:
                    self.logger.debug('Wait for {}s'.format(loop_time))
                    await asyncio.sleep(loop_time)

        finally:
            await session.close()
            self.logger.info('Watcher stop')
            self.logger.info('-----------------------')

def get_args():
    parser = argparse.ArgumentParser(description='Book Watcher')
    parser.add_argument('-s', '--shelf_number', required=True,
                        help='[REQUIRED]The class id of Book Shelf you watched.')
    parser.add_argument('-t', '--loop_time', type=int, default=3600,
                        help='Check loop interval [s]')

    parser.add_argument('-w', '--weixin_notify', action='store_true',
                        help='Notify with weixin')
    _weixin_notify = ('-w' in sys.argv) | ('--weixin_notify' in sys.argv)
    parser.add_argument('-wk', '--weixin_notify_key', required=_weixin_notify,
                        help='Get Weixin Notify Key from ServerChan')

    parser.add_argument('-m', '--mail_notify', action='store_true',
                        help='Notify with mail')
    _mail_notify = ('-m' in sys.argv) | ('--mail_notify' in sys.argv)
    parser.add_argument('-mh', '--mail_notify_host', required=_mail_notify,
                        help='SMTP mail host')
    parser.add_argument('-mu', '--mail_notify_user', required=_mail_notify,
                        help='SMTP mail username')
    parser.add_argument('-mp', '--mail_notify_pass', required=_mail_notify,
                        help='SMTP mail password')
    parser.add_argument('-ms', '--mail_notify_sender', required=_mail_notify,
                        help='SMTP mail sender')
    parser.add_argument('-mr', '--mail_notify_receiver', required=_mail_notify,
                        help='SMTP mail receiver')

    args = parser.parse_args()

    _weixin = Notifier.Weixin(args.weixin_notify_key) \
        if _weixin_notify else None
    _mail = Notifier.Mail(args.mail_notify_host, args.mail_notify_user, args.mail_notify_pass,
                          args.mail_notify_sender,
                          args.mail_notify_receiver) \
        if _mail_notify else None

    return namedtuple('args', ['shelf_number', 'loop_time', 'weixin', 'mail'])(
        args.shelf_number, args.loop_time, _weixin, _mail
    )

def init_logger():
    try:
        logging.config.fileConfig('src/hdu_library_watcher/logging.ini')
    except FileNotFoundError:
        os.mkdir('log')
        logging.config.fileConfig('src/hdu_library_watcher/logging.ini')
    return logging.getLogger('Watcher')

def main():
    args_tuple = get_args()
    logger = init_logger()
    notifier = Notifier(mail=args_tuple.mail, weixin=args_tuple.weixin)
    watcher = Watcher(notifier, logger)
    loop = asyncio.get_event_loop()
    loop.create_task(watcher.send_all_status_loop())
    loop.run_until_complete(watcher.check_loop(shelf=args_tuple.shelf_number, loop_time=args_tuple.loop_time))

if __name__ == '__main__':
    main()


from conans import ConanFile

class CctoolsTestConan(ConanFile):
    requires = 'llvm/5.0.2-1@vuo/stable'

    def imports(self):
        self.copy('*', dst='bin', src='bin')

    def test(self):
        self.run('bin/codesign_allocate -i bin/codesign_allocate -r -o codesign_allocate2')
        self.run('ls -l bin/codesign_allocate codesign_allocate2')
        self.run('file codesign_allocate2')
        self.run('./codesign_allocate2 -i codesign_allocate2 -r -o codesign_allocate3')
        self.run('ls -l codesign_allocate3')
        self.run('file codesign_allocate3')

        self.run('bin/lipo -info bin/lipo')

        self.run('bin/install_name_tool -add_rpath test codesign_allocate3')


# -*- coding: utf-8 -*-
"""
Created on Thu Feb  6 08:42:57 2020

@author: valramon
"""

import pandas as pd
import bokeh
from bokeh.plotting import figure
import numpy as np
from collections import defaultdict
from bokeh.models import ColumnDataSource, HoverTool, ResetTool, SaveTool, DataTable, TableColumn, BoxZoomTool
from bokeh.models.widgets import CheckboxGroup, Button, Select, Slider, Div, TextInput
from bokeh.models.ranges import Range1d
from bokeh.models import PolyDrawTool
from bokeh_callbacks import *
import os, glob, copy, pickle, platform

def create_map(n):
    """
        Esta función recibe un n predeterminado para crear el mapa de zonas. En caso de existir un archivo que ya
        contenga las zonas, simplemente lo carga y crea el mapa sobre el mismo.
    """
    #path donde se encuentre el csv con los datos de las zonas
    pathData = os.environ['USERPROFILE']+'/Downloads/bokeh_flask/bokeh_flask_vm/static/config/'

    archivosCsv = glob.glob(pathData + '*.csv')
    dic_cuadrados = defaultdict(list)

    archivosDb = glob.glob(pathData + '*.db')
    dictHistoricos = defaultdict(list)
    with open(archivosDb[-1], 'rb') as fileO:
        try:
            dataHist = pickle.load(fileO)
        except:
            dataHist = pickle.load(fileO, encoding = 'bytes')
        for i in range(len(dataHist)):
            dictHistoricos['id'].append(str(dataHist[i][1]))
            dictHistoricos['vehi'].append(str(dataHist[i][10]))
            dictHistoricos['progre'].append(str(round(dataHist[i][2], 2)))
            dictHistoricos['fecha'].append(str(dataHist[i][5].strftime("%Y/%m/%d - %H:%M:%S")))
            dictHistoricos['estado'].append(str(dataHist[i][3]))
    dataframe_historicos = pd.DataFrame(dictHistoricos)

    source_historicos = ColumnDataSource(dataframe_historicos)
    source_historicos_muestra = ColumnDataSource(copy.deepcopy(dataframe_historicos))
    del source_historicos.data['index']
    del source_historicos_muestra.data['index']
    keys = dictHistoricos.keys()
    columns = [
        TableColumn(field='id', title='ID'),
        TableColumn(field='vehi', title='Vehiculo'),
        TableColumn(field='progre', title='Progresiva'),
        TableColumn(field='fecha', title='Fecha'),
        TableColumn(field='estado', title='Estado')
    ]

    data_table = DataTable(source=source_historicos_muestra, columns=columns, width=300, height=400)
    data_table.visible = False
    x, y = [], []
    mayor = 0
    #carga los datos de las zonas en un diccionario, lo carga como string, por eso la separacion por comas
    if len(archivosCsv) != 0:
        if len(archivosCsv) > 1:
            for archivo in archivosCsv:
                try:
                    if int(archivo.split(' ')[-1].split('.')[0][1:-1]) > mayor:
                        mayor = int(archivo.split(' ')[-1].split('.')[0][1:-1])
                        archNuevo = archivo
                except:
                    pass
        else:
            archNuevo = archivosCsv[0]
        archCsv = open(archNuevo, 'r')
        data = archCsv.read()
        archCsv.close()
        keys = data.split('\n')[0].split(',')
        data = data.split('\n')[1:]
        dictData = defaultdict(list)
        j = 0
        for i in range(len(data)):
            for key in keys:
                dictData[key].append(data[i].split(',')[j])
                j += 1
            j = 0
        dic_datos_lineas = dictData
        dic_datos_lineas['RT_intensity'] = []
        largo_ducto = int(float(dic_datos_lineas['RT'][-1].split('pk')[-1]))
        n = len(dic_datos_lineas['RT'])
        lista_zonas = [(1.0,dic_datos_lineas['Intensity_tip'][k],dic_datos_lineas['Eventos_permitidos'][k]) for k in range(n)]
        for i in range(len(dic_datos_lineas['RT'])):
            dic_datos_lineas['RT_intensity'].append(np.arange(0, 50))
            dic_datos_lineas['RT'][i] = (float(dic_datos_lineas['RT'][i])) * np.ones(50)
            dic_datos_lineas['rel_int'][i] = float(dic_datos_lineas['rel_int'][i])
            x.append(dic_datos_lineas['RT'][i])
            y.append(dic_datos_lineas['RT_intensity'][i])
            if i != 0:
                dic_cuadrados['x'].append([[[dic_datos_lineas['RT'][i - 1][0], dic_datos_lineas['RT'][i - 1][0], dic_datos_lineas['RT'][i][0], dic_datos_lineas['RT'][i][0]]]])
            else:
                dic_cuadrados['x'].append([[[0, 0, dic_datos_lineas['RT'][i][0], dic_datos_lineas['RT'][i][0]]]])
            dic_cuadrados['y'].append([[[0, 50, 50, 0]]])
            if ("Ninguno" in dic_datos_lineas['Eventos_permitidos'][i]):
                dic_cuadrados['color'].append('White')
            else:
                dic_cuadrados['color'].append('Gray')
            dic_cuadrados['cont'].append(i)

    else:
        #si no existe un archivo, crea un diccionario con valores random para crear la figura del mapa
        largo_ducto = 30000
        dic_datos_lineas = defaultdict(list)
        int_fict=np.arange(50)
        order_loc=np.ones(50)

        n=int(n)
        choices=['Ninguno','Retroexcavadora/Camion']
        lista_zonas=[(1.0,'Zona '+str(k),np.random.choice(choices)) for k in range(n)]

        i = 0
        for scale, mz, evt_list in lista_zonas:
            dic_datos_lineas["RT"].append(order_loc*largo_ducto/n)
            dic_datos_lineas['ProgresivaIni'].append('pk'+str(round((order_loc[0] - 1)*int(largo_ducto/1000/n) + 338, 2)))
            dic_datos_lineas['ProgresivaFin'].append('pk'+str(round(order_loc[0]*int(largo_ducto/1000/n) + 338, 2)))
            dic_cuadrados['x'].append([[[(order_loc[0] - 1)*largo_ducto/n]]])
            dic_cuadrados['x'][i][0][0].append((order_loc[0] - 1)*largo_ducto/n)
            x.append(order_loc)
            order_loc=order_loc+1
            dic_cuadrados['x'][i][0][0].append((order_loc[0] - 1)*largo_ducto/n)
            dic_cuadrados['x'][i][0][0].append((order_loc[0] - 1)*largo_ducto/n)
            dic_datos_lineas["RT_intensity"].append(int_fict*scale)#norm(loc=120.4).pdf(RT_x) *
            y.append(int_fict*scale)
            dic_cuadrados['y'].append([[[0, 50, 50, 0]]])
            dic_datos_lineas['Eventos_permitidos'].append(evt_list)
            dic_datos_lineas['Intensity_tip'].append(mz)
            dic_datos_lineas['rel_int'].append(round(np.random.normal(loc=0.9,scale=0.01),2))
            if ("Ninguno" in evt_list):
                dic_datos_lineas['color'].append('Gray')
                dic_cuadrados['color'].append('White')
            else:
                dic_cuadrados['color'].append('Gray')
            if (len(evt_list.split('/')) == 1 and "Ninguno" not in evt_list):
                dic_datos_lineas['color'].append('Orange')
            if (len(evt_list.split('/')) == 2 and "Ninguno" not in evt_list):
                dic_datos_lineas['color'].append('Orange')
            if (len(evt_list.split('/')) == 3 and "Ninguno" not in evt_list):
                dic_datos_lineas['color'].append('Orange')
            dic_cuadrados['cont'].append(i)
            i += 1
        dic_datos_lineas['RT'][-1].fill(largo_ducto)
    i = 0
    coor = {'x': x, 'y':y}
    data = pd.DataFrame(dic_datos_lineas)
    data2 = pd.DataFrame(coor)
    zona_elegida = ['Zona 0']
    data_modifica = {'zona': zona_elegida}
    #transforma diccionarios a dataframes
    data_a_modificar = pd.DataFrame(data_modifica)
    data_respaldo = pd.DataFrame(copy.deepcopy(dic_datos_lineas))
    data_cuadrados = pd.DataFrame(dic_cuadrados)
    #Create the data sources for plotting the data
    #luego transforma los dataframes en data sources para poder trabajarlos en bokeh
    source = ColumnDataSource(data)
    source2_respaldo = ColumnDataSource(data2)
    source3 = ColumnDataSource(data_a_modificar)
    source_respaldo = ColumnDataSource(data_respaldo)
    source_cuadrados = ColumnDataSource(data_cuadrados)

    x_limit_left=largo_ducto+0.1

    #aqui setea las herramientas a utilizar en el mapa
    tools = [ResetTool(), SaveTool(), BoxZoomTool()]

    #crea la figura del mapa
    p = figure(plot_width=1200, plot_height=600,x_range=(0,x_limit_left), y_range=Range1d(0,49), tools = tools)

    #le incrusta la imagen del mapa y crea las lineas de las zonas
    p.image_url(url=['static/imgs/mapa_vm3.jpg'], x=0, y=0, w=x_limit_left, h=49,anchor='bottom_left')
    ml=p.multi_line(xs='RT', ys='RT_intensity',  # estas 4 lineas dibujan las lines de las zonas
                 line_width=5, line_color='color', line_alpha=0.7,
                 hover_line_color='pink', hover_line_alpha=1.0,
                 source=source)

    #setea las etiquetas para ver en la herramienta que sirve para cuando se pasa el mouse sobre una linea de zona
    hover_t=HoverTool(show_arrow=False, tooltips=[
        ('Zona', '@Intensity_tip'),
        ('Inicio de zona', '@ProgresivaIni'),
        ('Fin de zona', '@ProgresivaFin'),
        ('Eventos permitidos', '@Eventos_permitidos'),
        ('Intensidad relativa', '@rel_int')
    ], renderers = [ml])

    #crea los sombreados de las zonas
    poli = p.multi_polygons(xs='x', ys = 'y', color = 'color', line_alpha = 0,fill_alpha = 0.3, source = source_cuadrados)
    poli.visible = True

    #agrega la posibilidad de manipular las lineas de las zonas
    draw_tool_l1 = PolyDrawTool(renderers=[ml])

    p.add_tools(draw_tool_l1)

    p.xaxis.bounds=(0,n)
    p.yaxis.visible=False
    p.xaxis.visible=False

    oculta_zonas = ['Mostrar zonas']
    labels = [zona[1] for zona in lista_zonas]

    #crea los callbacks en javascript para cuando el script esta en ejecucion
    callback_checkbox = checkbox(source, source2_respaldo, source_respaldo, oculta_zonas)
    callback_guarda_zona = guarda_zona(source3)
    callback_evento = evento(source, source3, source_respaldo, source_cuadrados)
    callback_int = intensidad(source, source3, source_respaldo)

    eventos_perm = ["Ninguno", "Camioneta", "Camion", "Retroexcavadora"]

    #crea los distintos botones y herramientas a utilizar para el manejo del mapa, se crean en bokeh y funcionan con callbacks en javascript
    select = Select(title="Modificar zona:", value=source3.data['zona'][0], options=labels, callback = callback_guarda_zona)
    checkbox_group = CheckboxGroup(labels=oculta_zonas, active=[i for i in range (1)], callback=callback_checkbox)
    input_eventos = CheckboxGroup(labels=eventos_perm, active=[0], callback=callback_evento)
    slider_intensidad = Slider(start=0, end=1, value=0.5, step=.05, title="Intensidad relativa", callback = callback_int)

    callback_deshace_cambios = deshace_cambios(source, source_respaldo, source_cuadrados, input_eventos, slider_intensidad, eventos_perm, source3, draw_tool_l1)
    callback_download = download(source)
    text_input_prog_ini = TextInput(value='340', title='Progresiva de inicio')
    text_input_prog_fin = TextInput(value='342', title='Progresiva de fin')
    text_input_prog_fin.visible = False
    text_input_prog_ini.visible = False

    callback_confirma_zona = confirma_zona(source, source_respaldo, source_cuadrados, select, text_input_prog_ini, text_input_prog_fin)
    boton_confirma_zona = Button(label="Confirmar zona", button_type="success", callback = callback_confirma_zona)
    boton_confirma_zona.visible = False

    callback_agrega_zona = agrega_zona(text_input_prog_ini, text_input_prog_fin, boton_confirma_zona)

    callback_elimina_zona = elimina_zona(source, select)

    button = Button(label = "Rehacer mapa", button_type="success", callback = callback_deshace_cambios)

    button2 = Button(label="Aplicar cambios", button_type="warning", callback = callback_download)

    boton_agregar_zona = Button(label="Agregar zonas", button_type="warning", callback = callback_agrega_zona)

    boton_elimina_zona = Button(label="Eliminar zona", button_type="danger", callback = callback_elimina_zona)
    boton_elimina_zona.disabled = True

    div2 = Div(text="""
Ultimos 10 eventos de zona:
""", width=300, height=50)
    div2.visible = False

    callback_muestra_eventos_zona = muestra_eventos_zona(source, source_historicos, source_historicos_muestra, data_table, select, div2)

    boton_muestra_eventos_zona = Button(label='Mostrar ultimos eventos de zona', button_type='success', callback=callback_muestra_eventos_zona)
    boton_muestra_eventos_zona.disabled = True

    callback_mouse = mouse_drag(source, source_cuadrados, source3, select, poli, slider_intensidad, input_eventos, eventos_perm, source_respaldo, largo_ducto)
    callback_mouse_end = mouse_end(poli, source_cuadrados, source, largo_ducto, draw_tool_l1, boton_elimina_zona, source_respaldo)
    callback_mouse_move = mouse_move(source, source_respaldo, source_cuadrados)
    callback_tap = tap(source, source_respaldo, source3, source_cuadrados, slider_intensidad, input_eventos, eventos_perm, select, poli, draw_tool_l1, boton_elimina_zona, boton_muestra_eventos_zona, data_table, div2)

    #detecta eventos realizados con mouse en la figura para llamar a callbacks que modificar la figura del mapa
    p.js_on_event(bokeh.events.Tap, callback_tap)
    p.js_on_event(bokeh.events.Pan, callback_mouse)
    p.js_on_event(bokeh.events.MouseMove, callback_mouse_move)
    p.js_on_event(bokeh.events.PanEnd, callback_mouse_end)
    div = Div(text="""Eventos Permitidos:""", width=200, height=10)

    return p, checkbox_group, select, input_eventos, slider_intensidad, button, button2, div, div2, data_table, boton_muestra_eventos_zona, hover_t


import os
from .logger import Logger

log = Logger(os.getpid())

def normalize_grid_columns(grid, column_nos, defaults):
    for row_no, row in enumerate(grid):
        for i, col_no in enumerate(column_nos):
            cell = row[col_no]
            if cell == None:
                grid[row_no][col_no] = defaults[i]
                log.print(f"WARNING: setting column {col_no}, row {row_no} to '{defaults[i]}'")
    return grid


import math

def product(l):
    accu = 1.
    for e in l:
        accu *= e
    return accu

def boolean_or(l):
    for e in l:
        if l:
            return 1.
    return 0.

def boolean_et(l):
    for e in l:
        if not l:
            return 0.
    return 1.

functions_mapping = {
    '+': sum,
    '*': product,
    '-': (lambda x: -x[0]),
    'positif': (lambda x: float(x[0]>0)),
    'positif_ou_nul': (lambda x: float(x[0]>=0)),
    'null': (lambda x: float(x[0]==0)),
    'operator:>=': (lambda x: float(x[0]>=x[1])),
    'operator:=': (lambda x: float(x[0]==x[1])),
    'ternary': (lambda x: x[1] if x[0] else x[2]),
    'si': (lambda x: x[1] if x[0] else 0.),
    'inverse': (lambda x: 1/x[0] if x[0] else 0.),
    'max': max,
    'min': min,
    'inf': (lambda x: math.floor(x[0])),
    'arr': (lambda x: round(x[0])),
    'abs': (lambda x: abs(x[0])),
    'present': (lambda x: x[0] != 0.),
    'boolean:ou': boolean_or,
    'boolean:et': boolean_et,
    'dans': (lambda x: 1. if (x[0] in x[1:]) else 0.)
}