import numpy as np

from matplotlib.axes import Axes
from matplotlib.collections import PathCollection
from matplotlib.lines import Line2D
from matplotlib.projections import register_projection

class PickableAxes(Axes):
    """
    A subclass of matplotlib.axes._axes.Axes which support selection and action on data series.

    The axes currently support plotting methods the same as in a basic Axes except that
    a parent Run (i.e. DataFrame) can be supplied with the data. Corresponding data
    from the parent can be displayed for a particular point by clicking on it within
    the active Figure.

    The columns from the parent run which are shown when a daa point is selected are defined
    in the PickableAxes object's options.annotation_data attribute.

    Parameters
    ----------
    *args
        Positional arguments passed to the Axes constructor
    **kwargs
        Arbitrary keyword arguments passed to the Axes constructor

    Notes
    -----
    .. [1] To select multiple points within the same data series, hold Ctrl while clicking

    References
    ----------
    .. [1] matplotlib Custom Projections documentation:
           https://matplotlib.org/gallery/misc/custom_projection.html

    See Also
    --------
    .. [1] PickableAxesOptions
    .. [2] matplotlib.axes._axes.Axes
    """

    name = 'pickable'
    """
    The matplotlib projection name. The user will use this name to activate
    the PickableAxes functionality i.e. subplot(111, projection='pickable').
    """

    DATA_ARTISTS = (Line2D, PathCollection)
    """
    Supported artists (i.e. plot features) for pickable functionality.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.cla()

        self.handlers = {}
        self.options = PickableAxesOptions(self)

        self.figure.canvas.mpl_connect('pick_event', self.onpick)

    def loglog(self, *args, parent=None, indices=None, **kwargs):
        """
        Make a pickable plot with log scaling on both the x and y axes.

        Parameters
        ----------
        *args:
            Positional arguments passed to the Axes.loglog method
        parent: pygui.data.run.Run or DataFrame, optional. Default=None.
            The parent DataFrame that the x,y data came from.
        indices: list(int), optional
            The indices of each parent that the i'th data point corresponds to (if separate parent for each point).
        **kwargs:
            Arbitrary keyword arguments passed to the Axes.loglog method.

        Returns
        -------
        lines: list(Line2D)
            A list of Line2D object(s) representing the plotted data.

        Notes
        -----
        .. [1] A parent must be supplied in order for the data series to be pickable.
        """
        picker = self.options.picker if parent is not None else None
        lines = super().loglog(*args, picker=picker, **kwargs)
        self._add_artists(lines, parent)
        return lines

    def onpick(self, event):
        """
        Callback function for a pick event.

        Parameters
        ----------
        event: matplotlib.backend_bases.Event
            The pick event obejct.
        """

        # only act when supported objects are picked
        if any(isinstance(event.artist, o) for o in self.DATA_ARTISTS):
            artist = event.artist                   # the picked plot feature
            ind = self._get_closest_index(event)    # the index of the data series closest to the click
            key = event.mouseevent.key              # any keys that were pressed at the time of the click

            # get the artist handler obejct
            handler = self.handlers[artist]

            # TODO: IMPLEMENT ABILITY TO CHECK IF MULTIPLE LINES HAVE BEEN PICKED AND ONLY TAKE ACTION FOR ONE
            if key is None:
                # no keyboard keys held, simply select/deselect the artist
                handler.flip_selection_status(ind)
            elif 'control' in key or 'ctrl' in key:
                # allow multiple selections
                handler.select(ind)
            else:
                # simply select/deselect te artist
                handler.flip_selection_status()

    def plot(self, *args, parent=None, indices=None, **kwargs):
        """
        Make a pickable plot.

        Parameters
        ----------
        *args:
            Positional arguments passed to the Axes.loglog method
        parent: pygui.data.run.Run or DataFrame, optional. Default=None.
            The parent DataFrame that the x,y data came from.
        indices: list(int), optional
            The indices of each parent that the i'th data point corresponds to (if separate parent for each point).
        **kwargs:
            Arbitrary keyword arguments passed to the Axes.loglog method.

        Returns
        -------
        lines: list(Line2D)
            A list of Line2D object(s) representing the plotted data.

        Notes
        -----
        .. [1] A parent must be supplied in order for the data series to be pickable.
        """
        picker = self.options.picker if parent is not None else None
        lines = super().plot(*args, picker=picker, **kwargs)
        self._add_artists(lines, parent, indices)
        return lines

    def scatter(self, *args, parent=None, indices=None, **kwargs):
        """
        Make a pickable scatter plot.

        Parameters
        ----------
        *args:
            Positional arguments passed to the Axes.loglog method
        parent: pygui.data.run.Run or DataFrame, optional. Default=None.
            The parent DataFrame that the x,y data came from.
        indices: list(int), optional
            The indices of each parent that the i'th data point corresponds to (if separate parent for each point).
        **kwargs:
            Arbitrary keyword arguments passed to the Axes.loglog method.

        Returns
        -------
        paths: PathCollection
            Collection representing the plotted data.

        Notes
        -----
        .. [1] A parent must be supplied in order for the data series to be pickable.
        """
        picker = self.options.picker if parent is not None else None
        collections = super().scatter(*args, picker=picker, **kwargs)
        self._add_artists(collections, parent)
        return collections

    def semilogx(self, *args, parent=None, indices=None, **kwargs):
        """
        Make a pickable plot with log scaling on the x axis.

        Parameters
        ----------
        *args:
            Positional arguments passed to the Axes.loglog method
        parent: pygui.data.run.Run or DataFrame, optional. Default=None.
            The parent DataFrame that the x,y data came from.
        indices: list(int), optional
            The indices of each parent that the i'th data point corresponds to (if separate parent for each point).
        **kwargs:
            Arbitrary keyword arguments passed to the Axes.loglog method.

        Returns
        -------
        lines: list(Line2D)
            A list of Line2D object(s) representing the plotted data.

        Notes
        -----
        .. [1] A parent must be supplied in order for the data series to be pickable.
        """
        picker = self.options.picker if parent is not None else None
        lines = super().semilogx(*args, picker=picker, **kwargs)
        self._add_artists(lines, parent)
        return lines

    def semilogy(self, *args, parent=None, indices=None, **kwargs):
        """
        Make a pickable plot with log scaling on the y axis.

        Parameters
        ----------
        *args:
            Positional arguments passed to the Axes.loglog method
        parent: pygui.data.run.Run or DataFrame, optional. Default=None.
            The parent DataFrame that the x,y data came from.
        indices: list(int), optional
            The indices of each parent that the i'th data point corresponds to (if separate parent for each point).
        **kwargs:
            Arbitrary keyword arguments passed to the Axes.loglog method.

        Returns
        -------
        lines: list(Line2D)
            A list of Line2D object(s) representing the plotted data.

        Notes
        -----
        .. [1] A parent must be supplied in order for the data series to be pickable.
        """
        picker = self.options.picker if parent is not None else None
        lines = super().semilogy(*args, picker=picker, **kwargs)
        self._add_artists(lines, parent)
        return lines

    def _add_artists(self, artists, parent, indices=None):
        """
        Create a handler object for each artist and store them.

        Parameters
        ----------
        artists: list(matplotlib.artist.Artist)
            List of matplotlib artists (e.g. Line2D, PathCollection)
        parent: pygui.data.run.Run or DataFrame or None
            The parent DataFrame or None
        """
        if parent is not None:
            if isinstance(artists, PathCollection):
                artists = [artists]
            for artist in artists:
                self.handlers[artist] = self._create_artist_handler(artist, parent=parent, indices=indices)

    @classmethod
    def _create_artist_handler(cls, artist, *args, **kwargs):
        """
        Determine the type of the artist and create an appropriate handler object.

        Parameters
        ----------
        artist: matplotlib.artist.Artist
            The artist for which the handler is to be made.
        *args
            Positional arguments passed to the handler constructor.
        **kwargs
            Arbitrary keywords arguments passed to the handler constructor.

        Returns
        -------
        handler: _PickableArtistHandler or a subclass thereof
        """
        if isinstance(artist, Line2D):
            return _PickableLine2DHandler(artist, *args, **kwargs)
        elif isinstance(artist, PathCollection):
            return _PickablePathCollectionHandler(artist, *args, **kwargs)
        else:
            return _PickableArtistHandler(artist, *args, **kwargs)

    @staticmethod
    def _get_artist_data(artist):
        """
        Get the x,y data which the artist represents.

        Parameters
        ----------
        artist: matplotlib.artist.Artist
            the artist containing the data.

        Returns
        -------
        data: numpy.array (ndim=1), numpy.array (ndim=1)
        """
        if isinstance(artist, Line2D):
            return artist.get_data()
        elif isinstance(artist, PathCollection):
            data = artist.get_offsets()
            rows = data.shape[0]
            return data[:, 0].reshape((rows,)), data[:, 1].reshape((rows,))
        else:
            raise TypeError("Artist type not recognized. Supported types are 'Line2D' and 'PathCollection'")

    def _get_closest_index(self, event):
        """
        Get the data index closest to the pick event's mouse location.

        The event.ind attribute always seems to return the lower of the two indices
        that comprise a selected line segment, even if the mouse click was closer to
        the higher indexed point.

        This method checks both the returned index and the next higher index to determine
        which one is closer.

        Parameters
        ----------
        event: matplotlib.backend_bases.Event
            The pick event.

        Returns
        -------
        ind: int
            The index of the data closest to the click.
        """

        # get the lower index (or indices) that is returned by default
        indices = event.ind

        # get the full data series for the artist
        datax, datay = self._get_artist_data(event.artist)

        # check each returned index
        for i, index in enumerate(indices):
            # verify that the artist has a line and that the last item in the data series was not selected
            if event.artist.get_linestyle() is not None and index != len(datax)-1:
                # if so, append the next higher index
                indices = np.append(indices, indices[i]+1)

        # get the data corresponding to the lower and higher indices
        datax, datay = [datax[i] for i in indices], [datay[i] for i in indices]

        # get the mouse click location
        msx, msy = event.mouseevent.xdata, event.mouseevent.ydata

        # calculate the distance from the mouse click to the data
        dist = np.sqrt((np.array(datax)-msx)**2 + (np.array(datay)-msy)**2)

        # return the index corresponding to the smaller distance
        return indices[np.argmin(dist)]

class PickableAxesOptions(object):
    """
    An object to store options for PickableAxes for which the user has control.

    The attributes of a PickableAxesOptions object can be modified to adjust the
    settings for a particular axes. Alternatively, the class attributes can be modified
    to adjust the settings for all subsequently created PickableAxes, as the object
    attributes are set to the value of the class attributes upon initialization.

    Parameters
    ----------
    ax: matplotlib.axes._axes.Axes
        The axes for which the options correspond.
    """

    ANNOTATION_DATA = []
    """
    The column names from a parent object that will be displayed when a data point is selected.

    dtype: list(str)
    """

    ANNOTATION_PARAMS = {'xycoords': 'data',
                         'xytext': (10, 10),
                         'textcoords': 'offset points',
                         'fontsize': 8,
                         'bbox': {'boxstyle': 'round', 'fc': '0.8'},
                         'arrowprops': {'arrowstyle': '-|>'}}
    """
    Keywords arguments passed to the annotation box constructor when displaying data.

    dtype: dict{str: misc}
    """

    DRAGGABLE_ANNOTATIONS = True
    """
    Flag to allow the user to drag annotation boxes within a figure.

    dtype: bool
    """

    LINEWIDTH_DELTA = 2
    """
    Linewidth increment to indicate that a line has been selected.

    dtype: int
    """

    MARKERSIZE_DELTA = 6
    """
    Markersize delta to indicate that a data point has been selected.

    Note: not currently supported for scatter plots.

    dtype: int
    """

    PICKER = 5.
    """
    Picker setting for pickable artists. See https://matplotlib.org/users/event_handling.html.

    dtype: None, bool, float or function
    """

    def __init__(self, ax):
        self.ax = ax

        self.annotation_data       = self.ANNOTATION_DATA
        self.annotation_params     = self.ANNOTATION_PARAMS
        self.draggable_annotations = self.DRAGGABLE_ANNOTATIONS
        self.linewidth_delta       = self.LINEWIDTH_DELTA
        self.markersize_delta      = self.MARKERSIZE_DELTA
        self.picker                = self.PICKER

class _PickableArtistHandler(object):

    def __init__(self,
                 artist,
                 parent=None,
                 indices=None):
        self.artist = artist

        self.multiparent = False
        self.parent_indices = None
        self.parent = None
        self._store_parent(parent, indices)

        self.selected = False
        self.options = artist.axes.options

        self.annotations = []

        self._original_attributes = self._get_artist_attributes()
        self.selection_indicators = []

    def add_annotation(self, string, xy):
        annotation = self.artist.axes.annotate(string, xy, **self.options.annotation_params)
        if self.options.draggable_annotations:
            annotation.draggable()
        self.annotations.append(annotation)
        self.draw_idle()

    def add_selection_indicator(self, ind):
        x, y = self._get_data_coordinates(ind)
        attributes = self._selection_attributes
        # TODO: CLEAN THIS UP
        # TODO: FIGURE OUT HOW TO ADD INDICATOR FOR SCATTER PLOTS
        if 'linestyle' in attributes:
            del(attributes['linestyle'])
        if 'ls' in attributes:
            del(attributes['ls'])
        si = self.artist.axes.plot(x, y, linestyle=None, **attributes)
        self.selection_indicators += si
        self.draw_idle()

    def deselect(self, ind=None):
        raise NotImplementedError("This method must be overwritten by a subclass implemented for a specific type of" +
                                  "artist")

    def draw_idle(self):
        self.artist.figure.canvas.draw_idle()

    def flip_selection_status(self, ind=None):
        if self.selected:
            self.deselect(ind)
        else:
            self.select(ind)

    def remove_annotations(self):
        for annotation in self.annotations:
            annotation.remove()
        self.annotations = []
        self.draw_idle()

    def remove_selection_indicators(self):
        for si in self.selection_indicators:
            si.remove()
        self.selection_indicators = []
        self.draw_idle()

    def show_data(self, index, names=None):
        names = self.options.annotation_data if names is None else names
        data = self._get_parent_data(index=index, names=names)
        string = self._data_string(data)
        x, y = self._get_data_coordinates(index)
        self.add_annotation(string, (x, y))
        self.draw_idle()

    def select(self, ind=None):
        raise NotImplementedError("This method must be overwritten by a subclass implemented for a specific type of" +
                                  " artist")

    @property
    def _selection_attributes(self):
        raise NotImplementedError("This property must be overwritten by a subclass implemented for a specific type of" +
                                  " artist")

    @staticmethod
    def _data_string(data):
        s = ''
        nkeys = len(data.keys())
        for i, (name, value) in enumerate(data.items()):
            s += '%s: %s' % (name, str(value))
            if i != nkeys - 1:
                s += '\n'
        return s

    def _get_artist_attributes(self):
        return {}

    def _get_data_coordinates(self, ind):
        raise NotImplementedError("This method must be overwritten by a subclass implemented for a specific type of" +
                                  "artist")

    def _get_parent_data(self, index=None, names=None):
        names = self.options.annotation_data if names is None else names
        if self.multiparent:
            parent = self.parent[index]
            i = self.parent_indices[index]
            return {s: parent[s][i] for s in names}
        else:
            if index is None:
                return {s: np.array(self.parent[s]) for s in names}
            else:
                return {s: np.array(self.parent[s])[index] for s in names}

    def _store_parent(self, parent, indices=None):
        if any(isinstance(parent, o) for o in (list, tuple)):
            self.parent = parent
            self.parent_indices = indices
            self.multiparent = True
        else:
            self.parent = parent

class _PickableLine2DHandler(_PickableArtistHandler):

    def add_line_selection_indicator(self):
        attributes = self._selection_attributes
        self.artist.set_linewidth(attributes['lw'])
        self.draw_idle()

    def deselect(self, ind=None):
        self.remove_line_selection_indicator()
        self.remove_selection_indicators()
        self.remove_annotations()
        self.selected = False

    def remove_line_selection_indicator(self):
        lw = self._original_attributes['lw']
        self.artist.set_linewidth(lw)
        self.draw_idle()

    def select(self, ind=None):
        self.add_line_selection_indicator()
        if ind is not None:
            self.add_selection_indicator(ind)
            self.show_data(index=ind)
        self.selected = True

    @property
    def _selection_attributes(self):
        a = self._original_attributes.copy()
        a['lw'] = a['lw'] + self.options.linewidth_delta
        a['ms'] = a['ms'] + self.options.markersize_delta
        return a

    def _get_data_coordinates(self, ind):
        xdata, ydata = self.artist.get_data()
        if isinstance(ind, np.integer):
            return xdata[ind], ydata[ind]
        else:
            return np.array([xdata[i] for i in ind]), np.array([ydata[i] for i in ind])

    def _get_artist_attributes(self):
        return {'color': self.artist.get_color(),
                'ls': self.artist.get_ls(),
                'lw': self.artist.get_lw(),
                'marker': self.artist.get_marker(),
                'mec': self.artist.get_mec(),
                'mew': self.artist.get_mew(),
                'mfcalt': self.artist.get_mfcalt(),
                'ms': self.artist.get_ms()}

class _PickablePathCollectionHandler(_PickableArtistHandler):

    def deselect(self, ind=None):
        self.remove_selection_indicators()
        self.remove_annotations()
        self.selected = False

    def select(self, ind=None):
        if ind is not None:
            self.add_selection_indicator(ind)
            self.show_data(index=ind)
        self.selected = True

    @property
    def _selection_attributes(self):
        return {}

    def _get_artist_attributes(self):
        return {'edgecolor': self.artist.get_edgecolor(),
                'facecolor': self.artist.get_facecolor(),
                'fill': self.artist.get_fill(),
                'sizes': self.artist.get_sizes()}

    def _get_data_coordinates(self, ind):
        data = self.artist.get_offsets()
        if isinstance(ind, np.integer):
            return data[ind, 0], data[ind, 1]
        else:
            return np.array([data[i, 0] for i in ind]), np.array([data[i, 1] for i in ind])

register_projection(PickableAxes)


import discord
import emailer as emailer
from discord.ext import commands

client = commands.Bot(command_prefix = ",")
@client.event
async def on_ready():
    print("le bot est pret")

@client.command()
async def email(ctx,*,name):
    data = name.split(sep = " ")
    print(data)
    num = emailer.getcommandline()

    try:
        emailer.sendmail(data[0],data[1], num)
        await ctx.send(f'email had been sent to {data[0]}')
    except:
        await ctx.send(f"try typing in the following format : ,email (FullName) (Email)")

@client.command()
async def showcommands(ctx):
    file = open("commands.txt" , "r")
    data1= file.read()
    await ctx.send(data1)

client.run("OTYzMTY1NTI0MjYxNDg2NjUz.YlSHyw.oh64XYI5vepSx7W2Y0aBkmIdj9M")


# -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Jayesh Kariya <jayeshk@saltstack.com>`
'''
# Import Python libs
from __future__ import absolute_import

# Import Salt Testing Libs
from salttesting import skipIf, TestCase
from salttesting.mock import (
    NO_MOCK,
    NO_MOCK_REASON,
    MagicMock,
    patch)

from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

# Import Salt Libs
from salt.states import ntp

ntp.__salt__ = {}
ntp.__opts__ = {}

@skipIf(NO_MOCK, NO_MOCK_REASON)
class NtpTestCase(TestCase):
    '''
    Test cases for salt.states.ntp
    '''
    # 'managed' function tests: 1

    def test_managed(self):
        '''
        Test to manage NTP servers.
        '''
        name = 'coffee-script'

        ret = {'name': name,
               'result': False,
               'comment': '',
               'changes': {}}

        mock_lst = MagicMock(return_value=[])
        with patch.dict(ntp.__salt__, {'ntp.get_servers': mock_lst,
                                       'ntp.set_servers': mock_lst}):
            comt = ('NTP servers already configured as specified')
            ret.update({'comment': comt, 'result': True})
            self.assertDictEqual(ntp.managed(name, []), ret)

            with patch.dict(ntp.__opts__, {'test': True}):
                comt = ('NTP servers will be updated to: coffee-script')
                ret.update({'comment': comt, 'result': None})
                self.assertDictEqual(ntp.managed(name, [name]), ret)

            with patch.dict(ntp.__opts__, {'test': False}):
                comt = ('Failed to update NTP servers')
                ret.update({'comment': comt, 'result': False})
                self.assertDictEqual(ntp.managed(name, [name]), ret)

if __name__ == '__main__':
    from integration import run_tests
    run_tests(NtpTestCase, needs_daemon=False)


# S6.1: Design the View Data page of the multipage app.
# Import necessary modules
import streamlit as st
import pandas as pd
import numpy as np
# Define a function 'app()' which accepts 'car_df' as an input.
def app(car_df):
  st.markdown("
   VIEW DATA
",unsafe_allow_html = True)

  with st.beta_expander("View Dataset"):
  	st.table(car_df)

  st.markdown("""

Columns Description

    """,unsafe_allow_html = True)

  if st.checkbox("Show summary"):
    st.table(car_df.describe())

  beta_col1, beta_col2, beta_col3 = st.beta_columns(3)

  # Add a checkbox in the first column. Display the column names of 'car_df' on the click of checkbox.
  with beta_col1:
    if st.checkbox("Show all column names"):
        st.table(list(car_df.columns))

  # Add a checkbox in the second column. Display the column data-types of 'car_df' on the click of checkbox.
  with beta_col2:
  	if st.checkbox("View column data-type"):
  		st.table(car_df.dtypes)

  # Add a checkbox in the third column followed by a selectbox which accepts the column name whose data needs to be displayed.
  with beta_col3:
  	if st.checkbox("View column data"):
  		column_data = st.selectbox('Select column', tuple(car_df.columns))
  		st.write(car_df[column_data])


import os
from tkinter.tix import INTEGER
from conf import SAMPLE_INPUTS, SAMPLE_OUTPUTS
from moviepy.editor import *
from PIL import Image

source_path = os.path.join(SAMPLE_INPUTS, 'sample.mp4')

thumbnail_dir = os.path.join(SAMPLE_OUTPUTS, 'thumbnails')
thumbnail_per_frame_dir = os.path.join(SAMPLE_OUTPUTS, 'thumbnails-per-frame')
thumbnail_per_ms_dir = os.path.join(SAMPLE_OUTPUTS, 'thumbnails-per-ms')
thumbnail_per_halfs_dir = os.path.join(SAMPLE_OUTPUTS, 'thumbnails-per-halfs')

os.makedirs(thumbnail_dir, exist_ok=True)
os.makedirs(thumbnail_per_frame_dir, exist_ok=True)
os.makedirs(thumbnail_per_ms_dir, exist_ok=True)
os.makedirs(thumbnail_per_halfs_dir, exist_ok=True)

clip = VideoFileClip(source_path)
print (clip.reader.fps)
print (clip.reader.nframes)
print (clip.duration)

duration = clip.duration
max_duration = int(duration) + 1

for i in range(0, max_duration):
    frame = clip.get_frame(int(i))
    # print(frame)
    new_img_filepath = os.path.join(thumbnail_dir, f"{i}.jpg")
    #print (f"frame at {i} seconds saved at {new_img_filepath}")
    new_img = Image.fromarray(frame)
    new_img.save(new_img_filepath)

for i, frame in enumerate(clip.iter_frames()):
    #print(frame)
    new_img_filepath = os.path.join(thumbnail_per_frame_dir, f"{i}.jpg")
    #print (f"frame at {i} seconds saved at {new_img_filepath}")
    new_img = Image.fromarray(frame)
    new_img.save(new_img_filepath)

fps = clip.reader.fps
nframes = clip.reader.nframes
seconds = nframes/(fps*1.0)

for i, frame in enumerate(clip.iter_frames()):
    if i % fps == 0:
        current_ms = int((i/ fps) * 1000)
        # print(frame)
        new_img_filepath = os.path.join(thumbnail_per_ms_dir, f"{current_ms}.jpg")
        #print (f"frame at {i} seconds saved at {new_img_filepath}")
        new_img = Image.fromarray(frame)
        new_img.save(new_img_filepath)

for i, frame in enumerate(clip.iter_frames()):
    if i % int(fps / 2) == 0:
        current_ms = int((i / fps) * 10000)
        # print(frame)
        new_img_filepath = os.path.join(thumbnail_per_halfs_dir, f"{current_ms}.jpg")
        #print (f"frame at {i} seconds saved at {new_img_filepath}")
        new_img = Image.fromarray(frame)
        new_img.save(new_img_filepath)


import asyncio
import json
from typing import Any, AsyncGenerator, Coroutine, Generator, Optional

import requests
from fastapi import HTTPException
from pydantic import BaseModel, Field

from llmstudio.engine.providers.provider import ChatRequest, Provider, provider

class OllamaParameters(BaseModel):
    temperature: Optional[float] = Field(0.8, ge=0, le=1)
    num_predict: Optional[int] = Field(128, ge=1)
    top_p: Optional[float] = Field(0.9, ge=0, le=1)
    top_k: Optional[int] = Field(40, ge=0, le=500)

class OllamaRequest(ChatRequest):
    parameters: Optional[OllamaParameters] = OllamaParameters()

@provider
class OllamaProvider(Provider):
    def __init__(self, config):
        super().__init__(config)

    def validate_request(self, request: OllamaRequest):
        return OllamaRequest(**request)

    async def generate_client(
        self, request: OllamaRequest
    ) -> Coroutine[Any, Any, Generator]:
        """Generate an Ollama client"""
        try:
            return await asyncio.to_thread(
                requests.post,
                url="http://localhost:11434/api/generate",
                json={
                    "model": request.model,
                    "prompt": request.chat_input,
                    **request.parameters.dict(),
                },
                stream=True,
            )
        except requests.RequestException:
            raise HTTPException(
                status_code=500,
                detail="Ollama is not running",
            )

    async def parse_response(
        self, response: AsyncGenerator
    ) -> AsyncGenerator[str, None]:
        for line in response.iter_lines():
            if not line:
                continue
            chunk = json.loads(line.decode("utf-8"))
            if "error" in chunk:
                raise HTTPException(status_code=500, detail=chunk["error"])
            if chunk.get("done"):
                break
            yield chunk["response"]


import pandas as pd
import numpy as np
import re
from joblib import dump
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.linear_model import RidgeClassifier, LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from xgboost import XGBClassifier

SV_classifier = SVC()
KN_classifier = KNeighborsClassifier()
RF_classifier = RandomForestClassifier()
ridge_classifier = RidgeClassifier()
logit_regression = LogisticRegression()
MLP_classifier = MLPClassifier()
bag_classifier = BaggingClassifier()
NB_classifier = GaussianNB()
XGB_classifier = XGBClassifier()
gradient_boost_classifier = GradientBoostingClassifier()

SVC_param_grid = {'kernel': ['rbf', "linear", "polynomial"], 'decision_function_shape': ['ovo', 'ovr'], 'gamma': ["scale", "auto", 1e-3, 1e-4],
                  'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100]},
KNN_param_grid = {"n_neighbors": [2 * i for i in range(1, 41)], "weights": ["uniform", "distance"], "p": [1, 2], }
RFC_param_grid = {"n_estimators": [10 * i for i in range(5, 41)], "max_depth": [3, 8, 16, 32],
                  "min_samples_split": [2, 4, 6, 8], "min_samples_leaf": [1, 3, 5, 7]}
ridge_param_grid = {"alpha": [0.0001, 0.001, 0.01, 0.1, 1, 10],  "solver": ["svd", "cholesky", "sparse_cg", "lsqr", "sag"]}
logit_param_grid = {"C": [0.0001, 0.001, 0.01, 0.1, 1, 10],  "solver": ["liblinear", "newton-cg", "lbfgs", "sag", "saga"],  "max_iter": [3000]}
MLP_param_grid = {"hidden_layer_sizes": [(15, 10, 5, 2), (15, 30, 15, 2), (15, 15, 15, 2), (20, 15, 10, 10, 2)], "solver": ["lbfgs", "sgd", "adam"],
                  "alpha": [0.00001, 0.0001, 0.001, 0.01, 0.1], "learning_rate": ["constant", "invscaling", "adaptive"], "max_iter": [3000]}
bag_param_grid = {"n_estimators": [10 * i for i in range(2, 41)], "max_samples": [i for i in range(1, 15, 2)],
                  "max_features": [i for i in range(1, 15, 2)]}
NB_param_grid = {}
XGB_param_grid = {"booster": ["gbtree", "dart", "gblinear"], "eta": [0.001, 0.01, 0.1, 0.3], "max_depth": [3, 6, 8, 16],
                  "num_parallel_tree": [1, 3, 5], "subsample": [0.3, 0.5, 1]}
grad_boost_param_grid = {"loss": ["deviance", "exponential"], "learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.25],
                         "min_samples_split": [2, 4, 6, 8], "min_samples_leaf": [1, 3, 5, 7],
                         "max_depth": [3, 6, 8, 16], "n_estimators": [10 * i for i in range(5, 41)]}

def parse_name(name):
    """Given the name of each individual from the dataset, parses them into
     and returns their last name, title and other parts (inc. first names). """

    matches = re.match(r"(.*), (.*?)\. (.*)", name)
    last_name, title, other_name_part = matches.group(1, 2, 3)

    if title in ['Capt', 'Col', 'Don', 'Jonkheer', 'Major', 'Rev', 'Sir', 'Dr']:
        title = 'Mr'
    elif title in ['the Countess', 'Dona', 'Mme', 'Lady']:
        title = 'Mrs'
    elif title in ['Mlle', 'Ms']:
        title = "Miss"
    else:
        pass

    return last_name, title, other_name_part

# remove_outliers is only applied on the training set.
def remove_outliers(X, features, z_score):
    """We get the index of outliers, given each feature.
    If an index appears multiple times, that means it's an outlier of multiple features.
    Removes the individuals corresponding to each index from the dataset inplace. """

    outlier_indices = []
    for feature in features:
        mean = X[feature].mean()
        std_dev = X[feature].std()
        outlier_list_col = X[(X[feature] < mean - z_score * std_dev) |
                             (X[feature] > mean + z_score * std_dev)].index
        outlier_indices.extend(outlier_list_col)

    # Remove duplicate indices, if an individual is an outlier of multiple features.
    outlier_indices = list(set(outlier_indices))
    X.drop(outlier_indices, axis=0, inplace=True)

class FeaturesTransformer(BaseEstimator, TransformerMixin):
    def __init__(self):
        return

    def fit(self, X, y=None):
        return

    def transform(self, X, y=None):
        mean_age = X["Age"].mean()
        std_dev = X["Age"].std()
        X["Age"].fillna(np.random.randint(mean_age - std_dev, mean_age + std_dev), inplace=True)
        X.drop(["PassengerId", "Ticket", "Cabin"], axis=1, inplace=True)

        last_name_list = []
        titles = []
        for name in X["Name"]:
            last_name, title, other_name_part = parse_name(name)
            last_name_list.append(last_name)
            titles.append(title)

        last_name_list = np.array(last_name_list)
        titles = np.array(titles)

        X.drop("Name", axis=1, inplace=True)
        X["Title"] = titles

        median_fare = X["Fare"].median()
        X["Fare"].fillna(median_fare, inplace=True)

        X.dropna(subset=["Embarked"], inplace=True)

        # Add new attributes:
        X["RelativesOnboard"] = X["SibSp"] + X["Parch"]
        X.drop("SibSp", axis=1, inplace=True)
        X.drop("Parch", axis=1, inplace=True)

initial_feature_transformer = FeaturesTransformer()

num_attributes = ["Age", "Fare", "Pclass", "RelativesOnboard"]
numeric_transformer = Pipeline(steps=[
    ('scale', StandardScaler())
])
cat_attributes = ["Title", "Embarked", "Sex"]
cat_transformer = Pipeline(steps=[
    ('encode', OneHotEncoder())
])

preprocessing_transformer = ColumnTransformer(transformers=[
    ('num', numeric_transformer, num_attributes),
    ('cat', cat_transformer, cat_attributes)
])

def main():
    import time
    training_set = pd.read_csv("datasets\\train.csv")
    # Drop outliers:
    remove_outliers(training_set, ["Age", "Fare"], z_score=3)
    # Manipulating features.
    initial_feature_transformer.transform(training_set)

    # Seperating the Survived column from the rest of the dataset.
    y_train = training_set[["Survived"]].to_numpy().reshape(-1, 1)
    x_train = training_set.drop("Survived", axis=1)

    # Scaling each numerical attribute. Encoding each categorical attribute.
    x_train = preprocessing_transformer.fit_transform(x_train)

    classifiers = [SV_classifier, KN_classifier, RF_classifier, ridge_classifier, logit_regression,
                    MLP_classifier, bag_classifier, NB_classifier,
                   XGB_classifier, gradient_boost_classifier]

    param_grids = [SVC_param_grid, KNN_param_grid, RFC_param_grid, ridge_param_grid, logit_param_grid,
                    MLP_param_grid, bag_param_grid, NB_param_grid,
                    XGB_param_grid, grad_boost_param_grid]

    # Use Grid search each classification model
    # Grid search returns the best model, with select hyperparameters.
    # Model with the best score is our final model.
    score = 0
    grid_search = 0
    no_of_classifiers = len(classifiers)
    times = []
    list_grid_searches = []
    for index, (classifier, param_grid) in enumerate(zip(classifiers, param_grids)):
        print("K-fold cross-validation of {}... ({}/{})" .format(classifier, index+1, no_of_classifiers))

        last_time = time.time()
        current_grid_search = GridSearchCV(classifier, param_grid, cv=4,
                                       return_train_score=True, n_jobs= -1, verbose=1,
                                       scoring=["accuracy", "precision", "recall"], refit="accuracy")
        current_grid_search.fit(x_train, y_train.ravel())
        times.append(time.time() - last_time)

        if current_grid_search.best_score_ > score:
            grid_search = current_grid_search
            score = current_grid_search.best_score_

        list_grid_searches.append(grid_search)

    print("Time taken for K-fold cross-validation for each classifier (seconds):")
    for classifier, time in zip(classifiers, times):
        print("{0}: {1}" .format(classifier, time))
    print("\n")

    print("Best scores for each classifier type:")
    for each_grid_search in list_grid_searches:
        print("{0}, Score: {1}" .format(grid_search.estimator, each_grid_search.best_score_))
    print("\n")
    print("Final classifier: ")
    print("Best estimator:", grid_search.best_estimator_)
    print("Best parameters:", grid_search.best_params_)
    print("Score:", grid_search.best_score_)
    print("Index:", grid_search.best_index_, "\n")
    print("\n")
    cvresults = pd.DataFrame(grid_search.cv_results_)
    print(cvresults.iloc[grid_search.best_index_])
    print("\n")
    print("Use the model on the test set with 'test.py'")

    # Saves the grid_search variable with joblib.
    dump(grid_search, "final_model.joblib")

if __name__ == '__main__':
    main()


from dataclasses import dataclass
from networkx import DiGraph, contracted_edge, weakly_connected_components, subgraph, draw_networkx, nx_agraph, relabel_nodes, tree
import matplotlib.pyplot as plt
from random import randint
from networkx.drawing.nx_pydot import graphviz_layout
from matplotlib.backend_bases import MouseButton
import math

from matplotlib.widgets import TextBox

recursion_tree = DiGraph()

node_size = 300

count = 0

def read_and_inc():
    global count
    count += 1
    return count

@dataclass
class Node:
    graph: DiGraph
    hash: int
    is_component_node: bool

    def __init__(self, G: DiGraph, is_component: bool):
        self.hash = read_and_inc()
        self.graph = G
        self.is_component_node = is_component

    def __hash__(self) -> int:
        return self.hash

pos = {}

def enumerate_disconnected_graph(G: DiGraph) -> int:
    sum = 0

    full_graph_node = Node(G, True)

    recursion_tree.add_node(full_graph_node)

    for comp in weakly_connected_components(G):
        curr_graph = subgraph(G, comp)
        sum += enumerate_graph(curr_graph, full_graph_node)

    return sum

def enumerate_graph(G: DiGraph, parent_node: Node) -> int:
    curr_rec_tree_node = Node(G, False)
    recursion_tree.add_node(curr_rec_tree_node)
    recursion_tree.add_edge(parent_node, curr_rec_tree_node)

    # Recursion end
    if G.number_of_edges() == 0:
        return 1

    # Choose an edge
    edges = [(u, v) for u, v in G.edges]
    edge = edges[0]

    # Inline decision
    if G.in_degree(edge[1]) <= 1:
        lhs = enumerate_graph(
            contracted_edge(G, edge, self_loops=False, copy=True),
            curr_rec_tree_node)
    else:
        # If someone else calls this node, it needs to be duplicated before
        # inlining. This corresponds to a new graph with this edge removed and
        # edges from the fused not to all children of the node to be fused.
        new_graph = DiGraph()
        new_graph.add_nodes_from(G.nodes())
        new_graph.add_edges_from([e for e in edges if e != edge])
        new_graph.add_edges_from([(edge[0], e[1]) for e in edges
                                  if e[0] == edges[1]])
        lhs = enumerate_graph(new_graph, curr_rec_tree_node)

    # No Inline decision, this edge can be removed from the graph
    new_graph = DiGraph()
    new_graph.add_nodes_from(G.nodes())
    new_graph.add_edges_from([e for e in edges if e != edge])

    rhs = 0

    comps = weakly_connected_components(new_graph)
    comps = [c for c in comps]
    if len(comps) == 1:
        # If there is still one connected component, recurse into it
        rhs += enumerate_graph(subgraph(new_graph, comps[0]),
                               curr_rec_tree_node)

    else:
        # If there are multiple now (always exactly 2), created an inner node
        # and recurse
        inner_node = Node(new_graph, True)
        recursion_tree.add_node(inner_node)
        recursion_tree.add_edge(curr_rec_tree_node, inner_node)

        for comp in weakly_connected_components(new_graph):
            rhs += enumerate_graph(subgraph(new_graph, comp), inner_node)

    # This step needs all the computations from its children and one to evaluate
    # this choice
    return lhs + rhs + 1

def show_graph(G: DiGraph):

    #pos = graphviz_layout(G, prog="dot")
    draw_networkx(G, node_color="#a6cee3", font_size=10)
    plt.show()

def submit(index):
    i = int(index)
    plt.figure().clear()
    for n in recursion_tree.nodes:
        if n.hash == i:
            show_graph(n.graph)

fig, ax = plt.subplots()

if __name__ == "__main__":
    G = DiGraph(nx_agraph.read_dot("ex.ll.callgraph.dot"))

    labels = {}
    for i, n in enumerate(G.nodes):
        labels[n] = "N" + str(i)

    relabel_nodes(G, labels, copy=False)

    print(enumerate_disconnected_graph(G))

    rec_labels = {}
    for n in recursion_tree.nodes:
        rec_labels[n] = "N" + str(n.__hash__())

    node_colors = []
    for n in recursion_tree.nodes:
        node_colors.append("#b2df8a" if n.is_component_node else "#a6cee3")

    pos = graphviz_layout(recursion_tree, prog="dot")
    draw_networkx(recursion_tree,
                  ax=ax,
                  pos=pos,
                  labels=rec_labels,
                  node_color=node_colors,
                  font_size=10,
                  node_size=node_size)

    axbox = fig.add_axes([0.1, 0.05, 0.8, 0.075])
    text_box = TextBox(axbox, "Choose Node", textalignment="center")
    text_box.on_submit(submit)

    #d = ax.collections[0]

    plt.show()


import pathlib
# import classification
import os

# PACKAGE_ROOT = pathlib.Path(classification.__file__).resolve().parent
PACKAGE_ROOT = pathlib.Path(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
DATASET_DIR = PACKAGE_ROOT / 'dataset'
TRAINED_MODEL_DIR = PACKAGE_ROOT/'trained_model'

TESTING_DATA_FILE = DATASET_DIR/'test.csv'
TRAINING_DATA_FILE = DATASET_DIR/'train.csv'
TARGET = 'smoking'

FEATURES = ['age',
            'height(cm)',
            'weight(kg)',
            'waist(cm)',
            'eyesight(left)',
            'eyesight(right)',
            'hearing(left)',
            'hearing(right)',
            'systolic',
            'relaxation',
            'fasting blood sugar',
            'Cholesterol',
            'triglyceride',
            'HDL',
            'LDL',
            'hemoglobin',
            'Urine protein',
            'serum creatinine',
            'AST',
            'ALT',
            'Gtp',
            'dental caries']

DISCRETE_VARS =['age',
                'height(cm)',
                'eyesight(right)',
                'hearing(left)',
                'hearing(right)',
                'Urine protein',
                'dental caries']

CONTINOUS_VARS = ['weight(kg)',
                  'waist(cm)',
                  'eyesight(left)',
                  'systolic',
                  'relaxation',
                  'fasting blood sugar',
                  'Cholesterol',
                  'triglyceride',
                  'HDL',
                  'LDL',
                  'hemoglobin',
                  'serum creatinine',
                  'AST',
                  'ALT',
                  'Gtp']

MODEL_NAME = 'K-NEAREST_NEIGHBOR_CLASSIFIER'
PIPELINE_SAVE_FILE = f"{MODEL_NAME}_output_v"

ACCEPTABLE_MODEL_DIFFERNCE = 0.1


import requests # Get URL data
from bs4 import BeautifulSoup # Manipulate URL data
from pymongo import MongoClient
from datetime import timedelta, date
import os

# ad_decline takes a row of data from the JSE Market Activity list and converts it
# into a list of dictionaries formatted like:
# {ticker symbol, stock name, stock volume, closing price, percentage change}

def ad_decline(rows):
    data_list = []
    ticker_list = []
    name_list = []
    volume_list = []
    c_price_list = []
    p_change_list = []
    change_list = []
    for row in rows:
        data = row.find_all('td')
        ticker_list.append(data[0].text.strip())
        name_list.append(data[1].text.strip())
        volume_list.append(data[2].text.strip())
        c_price_list.append(data[3].text.strip())
        p_change_list.append(data[4].text.strip())
        change_list.append(data[5].text.strip())

    for i in range(len(ticker_list)):
        data_dict = {
            "ticker": ticker_list[i],
            "name": name_list[i],
            "volume": volume_list[i],
            "c_price": c_price_list[i],
            "price_change": p_change_list[i],
            "change": change_list[i]
        }
        data_list.append(data_dict)

    return(data_list)

# trading_firm takes a row of data from the JSE Market Activity list and converts it
# into a list of dictionaries formatted like:
# {ticker symbol, stock name, stock volume, closing price = 0, percentage change = 0}

def trading_firm(rows):
    data_list = []
    ticker_list = []
    name_list = []
    volume_list = []
    c_price_list = []
    for row in rows:
        data = row.find_all('td')
        ticker_list.append(data[0].text.strip())
        name_list.append(data[1].text.strip())
        volume_list.append(data[2].text.strip())
        c_price_list.append(data[3].text.strip())

    for i in range(len(ticker_list)):
        data_dict = {
            "ticker": ticker_list[i],
            "name": name_list[i],
            "volume": volume_list[i],
            "c_price": c_price_list[i],
            "price_change": "0",
            "change": "0"
        }
        data_list.append(data_dict)

    return(data_list)

# Scrapes the activity data and seperates them into three sets of table rows

def scrapeNews(date):
    url_string = "https://www.jamstockex.com/trading/trade-summary/?market=combined-market&date="
    test_page = requests.get(url_string + date)
    soup = BeautifulSoup(test_page.text, "html.parser")
    soup.prettify() # this gives HTML text for a given web page
    f = open("tabledata.txt", "a")

    try:
        p_items = soup.find_all("tbody")
        a_rows = p_items[0].find_all('tr')
        d_rows = p_items[1].find_all('tr')
        t_rows = p_items[2].find_all('tr')

        advancing = ad_decline(a_rows)
        declining = ad_decline(d_rows)
        firm = trading_firm(t_rows)

        data_list = {
            "advancing": advancing,
            "declining": declining,
            "trading_firm": firm,
            "date": date
        }

        stock_list = []
        for i in advancing:
            stock_list.append(i)
        for i in declining:
            stock_list.append(i)
        for i in firm:
            stock_list.append(i)

        outcome = date + " has been successfully scraped"
        print(outcome)
        return data_list

    except:
        # If there is no data it was potentially and holiday and should be skipped
        print("Potentially Holiday")
    return []

# Provides a list of dates inclusive of start and end date
def daterange(date1, date2):
    for n in range(int ((date2 - date1).days)+1):
        yield date1 + timedelta(n)

start_dt = date(2022,5,16)
end_dt = date(2022,5,17)

date_list = []
weekdays = [6,7]

# Removes weekdays from the date range
for dt in daterange(start_dt, end_dt):
    if dt.isoweekday() not in weekdays:
        date_list.append(dt.strftime("%Y-%m-%d"))

# Creates all of the list
def all_activity():

    activity_list = []
    for i in date_list:
        info = scrapeNews(i)
        if info != []:
            activity_list.append(info)

    return activity_list

# Insert Activities into the Activity Collection of MongoDB
def insert_activities():
    try:
        client = MongoClient(os.environ.get("DATABASE_URI"))
        print("Connection Successful")
        db = client.database

        activities = db.activities

        activity_list = all_activity()

        for i in activity_list:
            activities.insert_one(i)

        print("Successfully added activities")

    except:
        print("Connection Failed")

insert_activities()


import json
import math
import tvregdiff as tvrd
import dataFuncs
import numpy as np

class Point:
    def __init__(self, x, y, label = ''):
        if(isinstance(x, int) or isinstance(x, float)):
            self.x = x
        else:
            self.x = float(x)
        if(isinstance(y,int) or isinstance(y, float)):
            self.y = y
        else:
            self.y = float(y)
        self.label = label
    def close(self, other, epsilon):
        magSquared = (self.x-other.x)**2 + (self.y-other.y)**2
        return magSquared < epsilon**2
    def __getitem__(self, index):
        if index == 0: return self.x
        elif index == 1: return self.y
        elif index == 2: return self.label
        else: raise IndexError
    def __iter__(self):
        for a in [self.x, self.y]:
            yield a
    def __repr__(self):
        return 'Point' + str((self.x, self.y))
    def __add__(self, other):
        return Point(self.x+other.x, self.y+other.y)
    def __sub__(self, other):
        return Point(self.x-other.x, self.y-other.y)

class GraphData:
    def __init__(self, rawJSONString):
        decoded = json.loads(rawJSONString)
        axisData = decoded['axes']
        drawingData = decoded['drawing']
	    # axis data
        self.pixelHeight = int(axisData['yaxis']['pixels'])
        self.ymin = float(axisData['yaxis']['min'])
        self.ymax = float(axisData['yaxis']['max'])
        self.ystep = float(axisData['yaxis']['step'])
        self.pixelWidth = int(axisData['xaxis']['pixels'])
        self.xmin = float(axisData['xaxis']['min'])
        self.xmax = float(axisData['xaxis']['max'])
        self.xstep = float(axisData['xaxis']['step'])
        # drawing data
        tempPixels = drawingData['blackPixels']
        self.blackPixels = []
        self.xyPoints = []
        for bp in tempPixels:
            i, j = int(bp['i']), int(bp['j'])
            self.blackPixels.append(Point(i,j))
            x, y = self.xyFromIndex(i,j)
            self.xyPoints.append(Point(x,y))
        self.blackPixels.sort(key=lambda x: x[0])

    def xyFromIndex(self, i,j):
        i, j = float(i), float(j)
        x = (i/self.pixelWidth)*(self.xmax-self.xmin) + self.xmin
        y = self.ymax - (j/self.pixelHeight)*(self.ymax-self.ymin)
        return Point(x, y)

    def indexFromXY(self, x, y):
        x, y = float(x), float(y)
        i = (x - self.xmin)/float(self.xmax-self.xmin)*self.pixelWidth
        j = (self.ymax - y)/float(self.ymax-self.ymin)*self.pixelHeight
        return Point(int(round(i)), int(round(j)))

    def getFunctionList(self):
        if hasattr(self, 'functionList'):
            return self.functionList
        self.functionList = []
        d = {}
        for p in self.xyPoints:
            d[p.x] = d.get(p.x,[]) + [p.y]
        for x in d:
            y = float(sum(d[x]))/len(d[x])
            self.functionList.append(Point(x, y))
        self.functionList.sort(key=lambda p: p.x)
        return self.functionList

    def interpolate(self, x, l):
        ''' Linearly interpolate a list, l, of Points
        Return corresponding y
        '''
        # find point that is just a little smaller than the value x
        # using binary search
        if len(l) == 0: return None
        if len(l) == 1 or x < l[0].x: return l[0].y
        if x > l[-1].x: return l[-1].y

        lo, hi = 0, len(l) - 1
        while lo < hi:
            mid = (hi-lo)/2 + lo
            if l[mid].x < x:
                lo = mid + 1
            else:
                hi = mid - 1
        # lo and hi should be equal now
        if l[lo].x == x:
            return l[lo].y
        elif l[lo].x < x:
            x0, y0 = l[lo].x, l[lo].y
            x1, y1 = l[lo+1].x, l[lo+1].y
        else:
            x0, y0 = l[lo-1].x, l[lo-1].y
            x1, y1 = l[lo].x, l[lo].y
        # linear interpolation equation
        return y0 + (y1 - y0)*(x - x0)/float(x1 - x0)

    def getFunction(self):
        if hasattr(self, 'function'):
            return self.function
        self.function = lambda x: self.interpolate(x,self.getFunctionList())
        return self.function

    def gaussBlur(self, l):
        ans = []
        radius = 4
        sigma = 5.
        def gauss(r):
            return 1./math.sqrt(2.*math.pi*sigma**2.)*math.exp(-(r**2.)/(2.*sigma**2.))
        convolve = [gauss(ind) for ind in xrange(-radius, radius+1)]
        for i in xrange(len(l)):
            if l[i] == None: continue
            relevant = [(l[i+diff].y, convolve[radius + diff]) for diff in range(-radius, radius+1) if i+diff >= 0 and i+diff < len(l) and l[i+diff] != None]
            summ = sum([t[0]*t[1] for t in relevant])
            avg = summ/sum([t[1] for t in relevant])
            ans.append(Point(l[i].x, avg))
        return ans

    def getGaussFunctionList(self):
        if hasattr(self, 'gFunctionList'):
            return self.gFunctionList
        if len(self.getFunctionList()) < 3:
            self.gFunctionList = self.getFunctionList()
            return self.gFunctionList
        i = -1
        tempList = []
        for bp in self.blackPixels:
            newi = bp[0]
            if i != -1 and newi > i:
                tempList.extend([None for _ in range(i, newi)])
            if newi > i:
                i = newi
                x,_ = self.xyFromIndex(i, 0)
                y = self.getFunction()(x)
                tempList.append(Point(x,y))
        self.gFunctionList = self.gaussBlur(tempList)
        return self.gFunctionList

    def getGaussFunction(self):
        if hasattr(self, 'gaussFunction'):
            return self.gaussFunction
        self.gaussFunction = lambda x: self.interpolate(x,self.getGaussFunctionList())
        return self.gaussFunction

    def getSimpleDerivList(self):
        if hasattr(self, 'simpleDerivList'):
            return self.simpleDerivList
        self.simpleDerivList = []
        l = self.getFunctionList()
        for i in range(len(l)-1):
            x0, y0 = l[i].x, l[i].y
            x1, y1 = l[i+1].x, l[i+1].y
            self.simpleDerivList.append(Point(x0, float(y1-y0)/(x1-x0)))
        return self.simpleDerivList

    def getPixelDiff(self):
        if hasattr(self, 'pixelDiff'):
            return self.pixelDiff
        self.pixelDiff = []
        jovery = -self.pixelHeight/float(self.ymax-self.ymin)
        ioverx = self.pixelWidth/float(self.xmax-self.xmin)
        converter = jovery/ioverx
        for diff in self.getSimpleDerivList():
            i = self.indexFromXY(diff.x, 0)[0]
            pDiff = diff.y*converter
            self.pixelDiff.append(Point(i, pDiff))
        return self.pixelDiff

    def getSmoothDerivList(self):
        if hasattr(self, 'smoothDerivList'):
            return self.smoothDerivList
        if len(self.getFunctionList()) < 3:
            self.smoothDerivList = []
            return self.smoothDerivList
        # points must be evenly spaced apart
        step = self.xyFromIndex(2,0)[0] - self.xyFromIndex(1,0)[0]
        fl = self.getFunctionList()
        startx, endx = fl[0].x, fl[-1].x
        xs = []
        while startx <= endx:
            xs.append(startx)
            startx += step
        tempList = [self.getFunction()(x) for x in xs]
        der = tvrd.TVRegDiff(tempList, 50, 1e-1, scale='large', plotflag=0, diagflag=0)
        self.smoothDerivList = [Point(xs[i], der[i]) for i in range(len(xs))]
        return self.smoothDerivList

    def getSmoothDerivFunction(self):
        if hasattr(self, 'smoothDerivFunction'):
            return self.smoothDerivFunction
        self.smoothDerivFunction = lambda x: self.interpolate(x, self.getSmoothDerivList())
        return self.smoothDerivFunction

    def getAvgPixelIncrease(self, lo=-float('inf'), hi=float('inf')):
        l = self.getSimpleDerivList()
        if len(l) <= 0: return None
        avgIncr = sum([p.y for p in l if p.x >= lo and p.x <= hi])/float(len(l))
        pixPerY = self.canvasHeight/float(ymax-ymin)
        pixPerX = self.canvasWidth/float(xmax-xmin)
        return avgIncr*(pixPerY/pixPerX)


import pandas as pd
from utils import *
import regex
import traceback
from sentence_transformers import SentenceTransformer, util
import argparse
import os

os.environ["TOKENIZERS_PARALLELISM"] = "false"

def topic_pairs(topic_sent, all_pairs, threshold=0.5, num_pair=2):
    """
    Return the most similar topic pairs and the pairs that have been prompted so far
    - topic_sent: List of topic sentences (topic label + description)
    - all_pairs: List of all topic pairs being prompted so far
    - threshold: Threshold for cosine similarity
    - num_pair: Number of pairs to return
    """
    # Calculate cosine similarity between all pairs of sentences
    sbert = SentenceTransformer("all-MiniLM-L6-v2")
    embeddings = sbert.encode(topic_sent, convert_to_tensor=True)
    cosine_scores = util.cos_sim(embeddings, embeddings)
    over, pairs, dups = [], [], []
    for i in range(len(cosine_scores)):
        for j in range(len(cosine_scores)):
            if i != j and sorted([i, j]) not in dups:
                pairs.append({"index": [i, j], "score": cosine_scores[i][j]})
                dups.append(sorted([i, j]))

    # Sort and choose num_pair pairs with scores higher than a certain threshold
    pairs = sorted(pairs, key=lambda x: x["score"], reverse=True)
    count, idx = 0, 0
    while count < num_pair and idx < len(pairs):
        i, j = pairs[idx]["index"]
        if float(pairs[idx]["score"]) > threshold:
            if (sorted([topic_sent[i], topic_sent[j]]) not in all_pairs) and (
                topic_sent[i] != topic_sent[j]
            ):
                over.append([topic_sent[i], topic_sent[j]])
                all_pairs.append(sorted([topic_sent[i], topic_sent[j]]))
                count += 1
        idx += 1
    return [item for sublist in over for item in sublist], all_pairs

def merge_topics(
    topics_root,
    topics_node,
    prompt,
    deployment_name,
    temperature,
    max_tokens,
    top_p,
    verbose,
):
    """
    Prompt model to merge similar topics
    - topics_root: Root node of topic tree
    - topics_node: List of all nodes in topic tree
    - prompt: Prompt to be used for refinement
    - deployment_name: Model name
    - temperature: Temperature
    - max_tokens: Max tokens to generate
    - top_p: Top-p
    - verbose: Whether to print out results
    """
    # Get new pairs to be merged
    topic_sent = [
        f"[{topic.lvl}] {topic.name}: {topic.desc}" for topic in topics_root.descendants
    ]
    labels = [f"[{topic.lvl}] {topic.name}" for topic in topics_root.descendants]
    new_pairs, all_pairs = topic_pairs(
        topic_sent, all_pairs=[], threshold=0.5, num_pair=2
    )

    responses, removed, orig_new = [], [], {}
    # Pattern to match generations
    top_pattern = regex.compile(
        "^\[(\d+)\]([\w\s\-',]+)(?:[:\(\)\w\s\/])*?:([\w\s,\.\-\/;']+) \(((?:\[\d+\] [\w\s\-',]+(?:, )*)+)\)$"
    )
    # Pattern to match original topics being merged
    orig_pattern = regex.compile("(\[(?:\d+)\](?:[\w\s\-',]+)),?")

    while len(new_pairs) > 1:
        # Format topics to be merged in the prmpt
        inp, inp_label = [], []
        for topic in new_pairs:
            label = topic.split(":")[0]
            if label not in inp_label:
                inp.append(topic)
                inp_label.append(label)
        refiner_input = "\n".join(inp)
        refiner_prompt = prompt.format(Topics=refiner_input)
        if verbose:
            print(refiner_input)

        try:
            input_len = num_tokens_from_messages(refiner_input, "gpt-4")
            response = api_call(
                refiner_prompt, deployment_name, temperature, max_tokens, top_p
            )
            responses.append(response)
            merges = response.split("\n")
            for merge in merges:
                match = regex.match(regex.compile(top_pattern), merge.strip())
                if match:
                    lvl, name, desc = (
                        int(match.group(1)),
                        match.group(2).strip(),
                        match.group(3).strip(),
                    )
                    origs = [
                        t.strip(", ")
                        for t in regex.findall(orig_pattern, match.group(4).strip())
                    ]
                    orig_count = 0
                    add = False
                    if len(origs) > 1:
                        for node in topics_root.descendants:
                            if (
                                f"[{node.lvl}] {node.name}" in origs
                                and f"[{node.lvl}] {node.name}" != f"[{lvl}] {name}"
                            ):
                                orig_new[
                                    f"[{node.lvl}] {node.name}:"
                                ] = f"[{lvl}] {name}:"
                                if (
                                    f"[{node.lvl}] {node.name}: {node.desc}"
                                    in topic_sent
                                ):
                                    if verbose:
                                        print(
                                            f"Removing [{node.lvl}] {node.name}: {node.desc}\n"
                                        )
                                    topic_sent.remove(
                                        f"[{node.lvl}] {node.name}: {node.desc}"
                                    )
                                if f"[{node.lvl}] {node.name}" != f"[{lvl}] {name}":
                                    removed.append(f"[{node.lvl}] {node.name}")
                                orig_count += node.count
                                topics_node.remove(node)
                                node.parent = None
                                add = True
                        if add and f"[{lvl}] {name}" not in removed:
                            if (
                                f"[{lvl}] {name}: {desc}" not in topic_sent
                                and f"[{lvl}] {name}" not in labels
                            ):
                                new_node = Node(
                                    parent=topics_root,
                                    lvl=lvl,
                                    name=name,
                                    desc=desc,
                                    count=orig_count,
                                )
                                if verbose:
                                    print(f"Adding [{lvl}] {name}: {desc}\n")
                                topic_sent.append(f"[{lvl}] {name}: {desc}")
                                topics_node.append(new_node)
                            else:
                                if verbose:
                                    print(f"[{lvl}] {name} already exists!\n")
                                for node in topics_root.descendants:
                                    if f"[{node.lvl}] {node.name}" == f"[{lvl}] {name}":
                                        node.count += orig_count
        except:
            print("Error when calling API!")
            traceback.print_exc()
        print("--------------------")
        # Choose new pairs
        new_pairs, all_pairs = topic_pairs(
            topic_sent, all_pairs, threshold=0.5, num_pair=2
        )
    return responses, topics_root, orig_new

def remove_topics(topics_root, verbose, threshold=0.01):
    """
    Remove low-frequency topics from topic tree
    - topics_root: Root node of topic tree
    - verbose: Whether to print out results
    - Threshold: Percentage of all topic counts
    """
    topic_count = sum([node.count for node in topics_root.descendants])
    threshold = topic_count * threshold
    for node in topics_root.descendants:
        if node.count < threshold and node.lvl == 1:
            if verbose:
                print(f"Removing {node.name} ({node.count} counts)")
            node.parent = None
    return topics_root

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--deployment_name",
        type=str,
        help="model ('gpt-4', 'gpt-35-turbo', 'mistral-7b-instruct)",
    )
    parser.add_argument(
        "--max_tokens", type=int, default=500, help="max tokens to generate"
    )
    parser.add_argument("--temperature", type=float, default=0.0, help="temperature")
    parser.add_argument("--top_p", type=float, default=0.0, help="top-p")
    parser.add_argument(
        "--prompt_file",
        type=str,
        default="prompt/refinement.txt",
        help="file to read prompts from",
    )
    parser.add_argument(
        "--generation_file",
        type=str,
        default="data/output/generation_1.jsonl",
        help="file to read generation results from",
    )
    parser.add_argument(
        "--topic_file",
        type=str,
        default="data/output/generation_1.md",
        help="file to read topics from",
    )
    parser.add_argument(
        "--out_file",
        type=str,
        default="data/output/refinement_1.md",
        help="file to write merged topics to",
    )
    parser.add_argument(
        "--updated_file",
        type=str,
        default="data/output/refinement_1.jsonl",
        help="file to update merged topics to",
    )
    parser.add_argument(
        "--verbose", type=bool, default=False, help="whether to print out results"
    )
    parser.add_argument(
        "--remove", type=bool, default=False, help="option to remove minor topics"
    )
    parser.add_argument(
        "--mapping_file",
        type=str,
        default="data/output/refiner_mapping.json",
        help="file to write refined mapping to",
    )
    parser.add_argument(
        "--refined_again",
        type=str,
        default="refiner",
        help="Is this the second time refining?",
    )

    args = parser.parse_args()

    # Model configuration ----
    deployment_name, max_tokens, temperature, top_p = (
        args.deployment_name,
        args.max_tokens,
        args.temperature,
        args.top_p,
    )
    if deployment_name == "gpt-35-turbo":
        deployment_name = "gpt-3.5-turbo"

    # Load data ----
    topics_root, topics_node = generate_tree(read_seed(args.topic_file))

    # Prompting ----
    refinement_prompt = open(args.prompt_file, "r").read()
    responses, updated_topics_root, mapping = merge_topics(
        topics_root,
        topics_node,
        refinement_prompt,
        deployment_name,
        temperature,
        max_tokens,
        top_p,
        args.verbose,
    )
    if args.remove:
        updated_topics_root = remove_topics(
            updated_topics_root, args.verbose, threshold=0.01
        )

    if len(responses) > 0:
        # Writing updated topics ----
        with open(args.topic_file, "w") as f:
            print(tree_view(updated_topics_root), file=f)

        # Writing orig-new mapping ----
        try:
            current = open(args.mapping_file).read().strip().split("\n")
            if args.verbose:
                print(f"Writing to existing mapping file {args.mapping_file}")
        except:
            current = []
        with open(args.mapping_file, "w") as f:
            for key, value in mapping.items():
                current.append(f"{key} -> {value}")
            for topic in current:
                print(topic, file=f)

        # Updating generation.jsonl with new topics ----
        updated_responses = []
        df = pd.read_json(args.generation_file, lines=True)
        if args.refined_again == True:
            responses = df["refined_responses"].tolist()
        else:
            responses = df["responses"].tolist()
        for response in responses:
            splitted = response.split("\n")
            sub_list = []
            for s in splitted:
                for key, value in mapping.items():
                    if key != value and s.startswith(key):
                        s = s.replace(key, value)
                        if args.verbose:
                            print(f"Replacing {key} with {value}")
                sub_list.append(s)
            updated_responses.append("\n".join(sub_list))
        df["refined_responses"] = updated_responses
        df.to_json(args.updated_file, lines=True, orient="records")

if __name__ == "__main__":
    main()


import cv2
import time
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from skimage.morphology import medial_axis, skeletonize
cv2.destroyAllWindows()
def plot_color_image(image,title,colorscheme,file_ext,colorbar,path):
#    sizes=np.shape(image)
    my_dpi=96
    pix=700
    plt.figure(figsize=(pix/my_dpi, pix/my_dpi), dpi=my_dpi)
    plt.imshow(image,cmap=colorscheme)
    if colorbar == 'yes':
        plt.colorbar(fraction=0.046, pad=0.04)
#        plt.colorbar(orientation='horizontal')
    plt.title(title)
    plt.xticks([])
    plt.yticks([])
    print('saving %s.%s' % (title,file_ext,))
    plt.savefig('%s.%s' % (os.path.join(path,title),file_ext,),dpi=my_dpi, bbox_inches='tight')
    plt.show()

def plot_image(image,title,colorscheme,file_ext,colorbar,path):
#    sizes=np.shape(image)
    my_dpi=96
    pix=700
    plt.figure(figsize=(pix/my_dpi, pix/my_dpi), dpi=my_dpi)
    plt.imshow(image,cmap=colorscheme,vmin=0,vmax=255)
    if colorbar == 'yes':
        plt.colorbar(fraction=0.046, pad=0.04)
#        plt.colorbar(orientation='horizontal')
    plt.title(title)
    plt.xticks([])
    plt.yticks([])
    print('saving %s.%s' % (title,file_ext,))
    plt.savefig('%s.%s' % (os.path.join(path,title),file_ext,),dpi=my_dpi, bbox_inches='tight')
    plt.show()

def auto_canny(image, sigma=0.33):
	# compute the median of the single channel pixel intensities
	v = np.median(image)
 	# 1 standard deviation above and below median
	lower = int(max(0, (1.0 - sigma) * v))
	upper = int(min(255, (1.0 + sigma) * v))
	edged = cv2.Canny(image, lower, upper)

	# return the canny detection edged image
	return edged

plt.close('all')
add2x='.avi','_2x.avi'
big=0
path='C:/Users/Matthew/Documents/Image_Analysis/RBC_flow/Series'
cap1 = cv2.VideoCapture(os.path.join(path,'flow_crop'+add2x[big]))#_2x.avi'))
cap2 = cv2.VideoCapture(os.path.join(path,'flow_jpeg_crop'+add2x[big]))

n_frames = int(cap1.get(cv2.CAP_PROP_FRAME_COUNT))
fps = int(cap1.get(cv2.CAP_PROP_FPS))

cap1.set(cv2.CAP_PROP_POS_FRAMES, 0)
res, frame1 = cap1.read()
frame1 = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
cap2.set(cv2.CAP_PROP_POS_FRAMES, 0)
res, frame2 = cap2.read()
frame2 = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)
h,w = frame1.shape[:2]
cap1.release()
cap2.release()
frame1_prev = frame1.copy()
frame2_prev = frame2.copy()
cv2.imshow('1',frame1)
cv2.imshow('2',frame2)
#%%
kernel = np.ones((3,3),np.uint8)
kernel2 = np.zeros((11,11),np.uint8)
kernel2[5,5] = 3
boxFilter = np.ones((11,11))/125
kernel2=kernel2-boxFilter
cv2.destroyAllWindows()
path='C:/Users/Matthew/Documents/Image_Analysis/RBC_flow/Series'
cap1 = cv2.VideoCapture(os.path.join(path,'flow_crop'+add2x[big]))#_2x.avi'))
cap2 = cv2.VideoCapture(os.path.join(path,'flow_jpeg_crop'+add2x[big]))
fgbg = cv2.createBackgroundSubtractorMOG2(detectShadows=False)
w2=w*2
h2=h
out =cv2.VideoWriter(os.path.join(path,'norm_3T120.avi'),cv2.VideoWriter_fourcc(*'MJPG'),20,(w2,h2))

smear = 1
wrangle=11
thresh = 120
tol=0.01
n=[]
y=[]
for k in np.arange(0,n_frames,1):
    status1, frame1 = cap1.read()
    frame1 = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
    status2, frame2 = cap2.read()
    frame2 = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)

    if k==2:
        a=sum(sum(frame1-frame1_prev))
        b=sum(sum(frame2-frame2_prev))
    if wrangle ==1:
        for i in range(0,h):
            for j in range(0,w):
                if (frame1_prev[i,j]-frame1_prev[i,j]*tol)<= frame1[i,j] <(frame1_prev[i,j]+frame1_prev[i,j]*tol):
                    frame1[i,j]=frame1_prev[i,j]
                    y.append('yes')
                else:
                    frame1[i,j]=frame1[i,j]
                    n.append('no')
#                    frame2[i,j]=0
    if smear ==1:
        for i in range(0,h):
            for j in range(0,w):
                if frame1[i,j] <thresh:
                    frame1[i,j]=0
                    frame2[i,j]=0
#                else:
#                    frame1[i,j]=frame1[i,j]
##                    frame2[i,j]=0

    fgmask1 = fgbg.apply(frame1)
    frame1_2dFilter = cv2.filter2D(frame1,-1,kernel2)
    fgmask1_2dFilter = fgbg.apply(frame1_2dFilter)
#    fgmask = cv2.dilate(fgmask,kernel,iterations = 1)
#    fgmask = cv2.erode(fgmask,kernel,iterations = 1)
    fgmask1_2dFilter_med = cv2.medianBlur(fgmask1_2dFilter,3)

    fgmask2 = fgbg.apply(frame2)
    frame2_2dFilter = cv2.filter2D(frame2,-1,kernel2)
    fgmask2_2dFilter = fgbg.apply(frame2_2dFilter)

    f1track = np.hstack((frame1,fgmask1))#,frame1_2dFilter,fgmask1_2dFilter))
#    f2track = np.hstack((frame2,fgmask2,frame2_2dFilter,fgmask2_2dFilter))
#    f = np.vstack((f1track,f2track))
    f=np.dstack((f1track,f1track,f1track))
    cv2.imshow('frame',f)
    cv2.waitKey(20)
#    time.sleep(.05)  # secs
    out.write(f)
    print(k)
    frame1_prev = frame1.copy()
    frame2_prev = frame2.copy()

cap1.release()
cap2.release()
out.release()
cv2.destroyAllWindows()
##%%
#og=cv2.imread('original.png',0)
#cv2.imshow('og',og)


from django.urls import re_path as url
from . import views
from django.conf import settings
from django.conf.urls.static import static

app_name = 'pictures'

urlpatterns = [
    url(r'^login/', views.loginPage, name='login'),
    url(r'^logout/', views.logoutPage, name='logout'),

    url(r'^$', views.index, name='index'),
    url(r'^search/', views.search_results, name='search'),
    url(r'^location/(?P<location>\w+)/', views.image_location, name='location'),
    url(r'^add/', views.addPhoto, name='add'),

]
if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import requests
import json
import time
import re

URL = 'https://5ka.ru/api/v2/special_offers/'
headers = {'User-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
CAT_URL = 'https://5ka.ru/api/v2/categories/'

def x5ka(url, params):
    result = []
    while url:
        response = requests.get(url, headers=headers, params=params) if params else requests.get(url, headers=headers)
        params =None
        data = response.json()
        result.extend(data.get('results'))
        url = data.get('next')
        time.sleep(1)
    return result

def create_file_from_item(item):
    data = x5ka(URL, {'categories': item['parent_group_code']})
    filename = item['parent_group_name']
    filename = re.sub(r"[#%!@*/\n/\"]", "", filename)+'.json'
    with open(filename, 'w') as file:
        file.write(json.dumps(data))
        print(f"done: {filename}")

if __name__ == '__main__':
    categories = requests.get(CAT_URL, headers=headers).json()
    for item in categories:
        create_file_from_item(item)
    print("Готово, все файлы записаны!")


#
# @Function:This script is designed to trnasform sentences to a
#           100 dimensional vector
#
# @Input: Input a filename which indicates the place for your
#         text files that you would like to deploy transformation
#
# @Output: The text will be transformed and stored in to some
#          certain place
#

from gensim.models import Word2Vec
import keyword_parser
import numpy as np
import re
import sys

sys.dont_write_bytecode = True
params = eval(open("settings.txt").read())
model = Word2Vec.load(params['model_path'])
file_name = params['twitter_path']

#
# @Function:This function was used to break into word
#

def break_into_word(phrase_array):
    word_array = []
    for phrase in phrase_array:
        phrase_to_word = re.sub('[^a-zA-Z]', ' ', phrase.lower().strip()).split()
        word_array = word_array + phrase_to_word
    return word_array

def generate_words_array(file_name):
    raw_data = []
    w = open(file_name,'r')
    for line in w.readlines():
        newline = keyword_parser.get_keywords(line, ['NP'])
        newline = break_into_word(newline)
        raw_data.append(newline)
    w.close()
    return raw_data
#
# @Function: From this function when we input a sentence
#            in the word list form we would get a vector
#            back which is the vector of this sentence
#

def sentence_to_vec():
    sentence_array = generate_words_array(file_name)
    return_ans = []
    for sentence in sentence_array:
        sentence_vector = []
        for word in sentence:
            try:
                sentence_vector.append(model.wv[word])
            except:
                continue
        sentence_vector = np.array(sentence_vector)
        v = sentence_vector.sum(axis = 0)
        ans =  v/np.sqrt((v ** 2).sum())
        return_ans.append(ans)
    return return_ans

sentence_to_vec()


from django.shortcuts import render, get_object_or_404, redirect
from .models import Project
from django.contrib.auth.decorators import login_required

def home(request):
    projects = Project.objects
    return render(request, 'projects/home.html', {'projects':projects})

def detail(request, project_id):
    project = get_object_or_404(Project, pk=project_id)
    return render(request, 'projects/detail.html', {'project':project})

@login_required(login_url="/accounts/login")
def upvote(request, project_id):
    if request.method=='POST':
        project = get_object_or_404(Project, pk=project_id)
        project.votes_total += 1
        project.save()
        return redirect('/projects/' + str(project.id))

@login_required(login_url="/accounts/login")
def upvote1(request, project_id):
    while request.method=='POST':
        project = get_object_or_404(Project, pk=project_id)
        project.votes_total += 1
        project.save()
        return redirect('home')

@login_required(login_url="/accounts/signup")
def enquiry(request, project_id):
    if request.method=='POST':
        project = get_object_or_404(Project, pk=project_id)
        return render(request, 'projects/enquiry.html')

def about(request):
    return render(request, 'projects/about.html')


import matplotlib.pyplot as plt

from calculator import HUCalculator

mu = lambda x: 1
beta = lambda x: 50 * pow(x, 2)
sigma = lambda x: 40 * pow(x, 2)
f = lambda x: 40 * pow(x - 0.3, 5)

N = 5
a = -1
b = 1

calc = HUCalculator(N, a, b, mu, beta, sigma, f)
calc = calc.calc_until(5, 50)

print("N\t| Норма оцінювача\t\t| Норма розв'язку")
for i in range(len(calc.Ns)):
    print(f"{calc.Ns[i]}\t| {calc.eNs[i]}\t| {calc.uNs[i]}")

curr_pos = 0

fig, ax = plt.subplots()
ax.plot(calc.get_x(), calc.get_y(), marker='o')
plt.xlabel('x')
plt.ylabel(f'u')
plt.show()


# -*- coding: utf-8 -*-

from django.contrib import messages
from uuid import uuid4
from Contract.models import Stage, Attache
from Contract.forms import FormStage

######################################################################################################################

def change_stage(request, contract, status):
    stage = Stage(contract=contract, status=status)
    formset_stage = FormStage(request.POST, request.FILES, instance=stage)
    if formset_stage.is_valid():
        formset_stage.save()
        for file_attache in request.FILES.getlist('files'):
            attache = Attache(name=file_attache.name, stage=stage)
            attache.file.save(uuid4().hex, file_attache)
            attache.save()
            if not status.next_status:
                contract.closed = True
                contract.save()
        return True
    else:
        return False

######################################################################################################################


import numpy as np
import os
from PIL import Image

# root_dir="C:/Users/sh/Desktop/data"
root_dir = "C:/Users/huili/Desktop/fruitdatas"
image_folders = os.listdir(root_dir)
floders_num = 0
for i in image_folders:
    image_list = os.listdir(root_dir + '/' + i)
    for j in image_list:
        # if os.path.splitext(j)[1] != ".png":
        if os.path.splitext(j)[1] != ".jpg":
            image_list.remove(j)
        else:
            pass
    print(len(image_list))
    image = np.zeros((len(image_list), 480, 640, 3))
    num = 0
    for j in image_list:
        im = Image.open(root_dir + '/' + i + '/' + j)
        # print(np.shape(np.array(im)))
        image[num, :, :, :] = np.array(im)
        num = num + 1
    np.save('image/' + image_folders[floders_num] + '.npy', image)
    floders_num = floders_num + 1
    print(np.shape(image))

print('finish')


from re import I

from uuid import uuid4

def renderPlaceholder(placeholder: str, value: str,filename: str):
    with open(filename, 'r') as f:
        content = f.read()
        return content.replace(placeholder, value)

xsrf_token = str(uuid4())
addedTokenPage = renderPlaceholder("{{token}}", xsrf_token, "index.html")
print("Adding token into page...")
print(addedTokenPage)


"""Доработаем задачи 3 и 4. Создайте класс Project, содержащий атрибуты – список пользователей проекта и админ проекта.
Класс имеет следующие методы:
Классовый метод загрузки данных из JSON файла (из второй задачи 8 семинара)
Метод входа в систему – требует указать имя и id пользователя.
Далее метод создает пользователя и проверяет есть ли он в списке пользователей проекта.
Если в списке его нет, то вызывается исключение доступа.
Если пользователь присутствует в списке пользователей проекта, то
пользователь, который входит получает его уровень доступа и становится администратором.
Метод добавление пользователя в список пользователей. Если уровень пользователя меньше,
чем ваш уровень, вызывайте исключение уровня доступа.
Доработайте классы исключения так, чтобы они выдали
подробную информацию об ошибках.
Передавайте необходимые данные из основного кода
проекта."""
import json

from sem_13.task_13_3 import AccessExeption, LevelExeption
from sem_13.task_13_4 import User

class Project:

    data = None

    def __init__(self, users_list: list, admin=None, file_path= "data.json"):
        self.file_path = file_path
        self.users_list = users_list
        self.admin = admin

    @classmethod
    def load_from_json(cls, file_path):
        cls.file_path = file_path
        users_list = []
        try:
            with open(file_path, "r", encoding="utf8") as file:
                cls.data = json.load(file)
            for level, users in cls.data.items():
                for id_, name in users.items():
                    users_list.append(User(id_, name, level))
        except FileNotFoundError:
            pass

        return Project(users_list)

    def login_user(self):
        user_input = input("Введите Имя и id для входа ")
        name, id_ = user_input.strip().split(" ")
        user = User(id_, name, 0)
        if user not in self.users_list:
            AccessExeption(user)
        for each in self.users_list:
            if each == user:
                self.admin = each
                print("Вы вошли")
                print(self.admin)

    def add_user(self):
        user_input = input("Для добавления нового пользователя введите имя, идентификатор, уровень доступа: ")
        name, id_, level = user_input.split(" ")
        self.users_list.append(User(id_, name, level))
        if level < self.admin.level:
            raise LevelExeption(self.admin.level)
        if id_ not in [user.user_id for user in self.users_list]:
            self.data.setdefault(level, {id_: name})[id_] = name
        try:
            with open(self.file_path, "w", encoding="utf8") as file:
                json.dump(self.data, file, ensure_ascii=False, indent=1)
        except BaseException():
            pass

    def __str__(self):
        return f"{[user for user in self.users_list]}"

    def __repr__(self):
        return f"{[user for user in self.users_list]}"

if __name__ == '__main__':
    project = Project.load_from_json("D:\\dev\\study\\Immersion _in_Python\\sem_8\\task_8_2.json")
    # project = Project.load_from_json("data.json")
    print(project)
    project.login_user()
    print("+++ ", project.admin, project.admin)
    project.add_user()
    print("----", project)


import numpy as np
import nolds

def mle(x: np.ndarray) -> float:
    """ Maximum Lyapunov Exponent
    :param x: 1-d numeric vector
    :return: numeric scalar
    """
    k = int(np.sqrt(len(x)))

    try:
        out = nolds.lyap_r(data=x,
                           emb_dim=k,
                           trajectory_len=k,
                           min_neighbors=k)
    except (ValueError, np.linalg.LinAlgError, AssertionError) as e:
        out = np.nan

    return out


import argparse
import glob
from multiprocessing.pool import ThreadPool
import numpy as np
import os
import random
import shutil
import subprocess
import tarfile
import threading
import sys
sys.path.append('./')
from network import network

def parse_args():
  argv = sys.argv[1:]
  if '--' in argv:
    remaining_args = argv[argv.index('--') + 1:]
    argv = argv[:argv.index('--')]
  else:
    remaining_args = []
  args = parser.parse_args(argv)
  return args, remaining_args

parser = argparse.ArgumentParser()
parser.add_argument('--results_dir', help='Directory to store results in/upload results from', default='eval_results/', type=str)
parser.add_argument('--team', help='Registered team name', default='', type=str)
parser.add_argument('--addr', help='Address of the leaderboard server', default='http://6829fa18.csail.mit.edu', type=str)
parser.add_argument('--upload', default=False, action='store_true')
parser.add_argument('--run', default=False, action='store_true')
args, remaining_args = parse_args()

def run_in_threadpool(f, args):
    pool = ThreadPool()
    lens = pool.map(f, args)
    pool.close()
    pool.join()
    return lens

def subprocess_cmd(command, lock=threading.Lock()):
    process = subprocess.Popen(command,
                               stdout=sys.stdout,
                               stderr=sys.stderr,
                               shell=True)
    process.wait()
    return process

def get_length(trace):
    with open(trace, 'r') as f:
        return sum(1 for line in f)

def cmd_gen(trace, start_index, results_dir):
    return 'python sim/run_exp.py -- --mm-trace=%s --results-dir=%s --mm-start-idx=%d -- %s' % (
        trace, results_dir, start_index,
        ' '.join(remaining_args))

def run():
    import urllib

    np.random.seed(1)

    if not os.path.isdir('/tmp/6829-smp'):
        r = urllib.urlopen('http://6829fa18.csail.mit.edu:8080/smp.tar.gz')
        with open('/tmp/6829-smp.tar.gz', 'wb') as f:
            f.write(r.read())
        tar = tarfile.open('/tmp/6829-smp.tar.gz', 'r:*')
        tar.extractall('/tmp/6829-smp')
        tar.close()
    assert(os.path.isdir('/tmp/6829-smp'))

    trace_dir = '/tmp/6829-smp/heldout/test'
    n_runs = 64

    traces = []
    cmds = []
    for fname in os.listdir(trace_dir):
      print(fname)
      if fname.endswith('.log'):
        traces.append(os.path.join(trace_dir, fname))
    np.random.shuffle(traces)

    while len(traces) < n_runs:
      traces.append(np.random.choice(traces))
    traces = traces[:n_runs]

    # rescale the throughputs.
    sample_throughputs = np.random.uniform(.3, 4, n_runs)
    rescaled_traces = []
    os.system('mkdir -p /tmp/rescaled_traces/test/')
    for trace, thr in zip(traces, sample_throughputs):
      rescaled_trace = '/tmp/rescaled_traces/%s/thr_%.2f_%s' % (
          'test', thr, os.path.basename(trace))
      network.trace_with_target(trace, rescaled_trace, thr)
      rescaled_traces.append(rescaled_trace)

    traces = rescaled_traces

    lens = run_in_threadpool(get_length, traces)
    # sample pointer from the first three-quarters of the trace.
    start_indices = [int(l * np.random.random() * .75) for l in lens]

    for trace, start_index in zip(traces, start_indices):
      cmd = cmd_gen(
          trace, start_index,
          os.path.join(args.results_dir, 'leaderboard', 'test',
                       os.path.basename(trace).split('.log')[0]))
      cmds.append(cmd)

    run_in_threadpool(subprocess_cmd, cmds)

def upload():
    import requests

    # Check if the results directory is there
    if not os.path.exists(args.results_dir):
        print("Results directory '%s' doesn't exist. Run the experiment to create directory" % args.results_dir)
        return

    if args.team == '':
        print("Please specify team name using '--team'")
        return

    # If the tarball already exists, delete
    tfname = os.path.join(args.results_dir, 'results.tar.gz')
    if os.path.exists(tfname):
        os.remove(tfname)

    # Put folder into a tarball
    print("Writing tarfile...")
    tar = tarfile.open(tfname, 'w:gz')
    tar.add(os.path.join(args.results_dir, 'leaderboard', 'test'))
    tar.close()

    with open(tfname, 'rb') as f:
        files = {"results": f}
        data = {"team": args.team}
        addr = args.addr + '/upload_file'
        r = requests.post(addr, data=data, files=files)
        print(r.content.decode())

    # Example using curl
    # curl localhost:8888/upload_file -Fteam=myteam2 -Fresults='@eval_results/results.tar.gz'

if args.run:
    run()
if args.upload:
    upload()


#from tgpirobot.tgpirobot import TgPiRobot, print_help
from tgpirobot import TgPiRobot, print_help, instally
import sys
import os
from rich.console import Console
from rich.progress import track, Progress
import subprocess
from pyrogram.errors.exceptions.unauthorized_401 import AuthKeyUnregistered
try:
    def update():
        print("Updating tgpirobot...")

        with Progress() as progress:
            task = progress.add_task("[cyan]Progress...", total=100)

            pip_process = subprocess.Popen(["pip", "install", "--upgrade", "tgpirobot"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            for i in range(1, 101):
                progress.update(task, completed=i)
                time.sleep(0.3)

            pip_process.communicate()

        print("Update complete.")

    console = Console()

    def main():
        bot = TgPiRobot()
        if len(sys.argv) > 1:
            arg = sys.argv[1]
            if arg in ["--help", "-h", "help"]:
                print_help()
                console.theme = None
            elif arg in ["--run", "-r", "run"]:
                try:
                    bot.run()
                except ImportError as e:
                    print("")
                    console.log(f"\n[bold red]Error:[/bold red] {e}")
                    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")
                except Exception as e:
                    print("")
                    console.log(f"\n[bold red]Error:[/bold red] {e}")
                    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")
                except AuthKeyUnregistered as e:
                    print("")
                    console.log(f"\n[bold red]Error:[/bold red] {e}")
                    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")
                except OperationalError as e:
                    pirint("")
                    console.log(f"\n[bold red]Error:[/bold red] {e}")
                    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")

                console.theme = None
            elif arg in ["--update", "-u", "update"]:
                update()
                console.theme = None
            elif arg in ["--install", "-i", "install"]:
                instally()
            elif arg in ["--del", "-d", "del"]:
                console.log(f"[bold yellow]Debug:[/bold yellow] Removing old session file")
                os.system("rm $PREFIX/bin/tgpirobot.session")
                time.sleep(1)
                console.log(f"[bold yellow]Done:[/bold yellow] Removed old session file, now run by tgpirobot -r")
            else:
                print("Invalid option : tgpirobot -h for help (If still error then try tgpirobot -d)")
        else:
            print("No option provided : tgpirobot -h for help (If still error then try tgpirobot -d)")

    if __name__ == "__main__":
        main()
except ImportError as e:
    console.log(f"[bold red]Error:[/bold red] {e}")
    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")
except Exception as e:
    console.log(f"[bold red]Error:[/bold red] {e}")
    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")
except AuthKeyUnregistered as e:
    console.log(f"[bold red]Error:[/bold red] {e}")
    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")
except OperationalError as e:
    console.log(f"[bold red]Error:[/bold red] {e}")
    console.log("[bold red]Run[/bold red] [bold green]tgpirobot -d[/bold green][bold red] and then[/bold red] [bold green]tgpirobot -r[/bold green]")


import pytest
from cognite.client import data_modeling as dm

from cognite.pygen._core.generators import SDKGenerator
from cognite.pygen._generator import CodeFormatter
from tests.constants import APM_SDK, EXAMPLES_DIR

@pytest.fixture
def sdk_generator(apm_data_model: dm.DataModel[dm.View]) -> SDKGenerator:
    return SDKGenerator(
        APM_SDK.top_level_package,
        APM_SDK.client_name,
        apm_data_model,
    )

def test_generate_sdk(sdk_generator: SDKGenerator, code_formatter: CodeFormatter) -> None:
    # Act
    sdk = sdk_generator.generate_sdk()

    # Assert
    for file_path, actual in sdk.items():
        expected_location = EXAMPLES_DIR / file_path
        assert expected_location.exists()
        expected = expected_location.read_text()

        actual = code_formatter.format_code(actual)
        assert actual == expected


import streamlit as st
from snowflake.snowpark import Session

def icon(emoji: str):
    """Shows an emoji as a Notion-style page icon."""
    st.write(
        f'{emoji}',
        unsafe_allow_html=True,
    )

st.set_page_config("Snowpark & PySpark support", "🏂")
icon("🏂")
"""
# Snowpark & PySpark support

[Release 1.16.0](https://docs.streamlit.io/library/changelog#version-1160) of Streamlit
adds support for [Snowpark](https://docs.snowflake.com/en/developer-guide/snowpark/index.html[)
and [PySpark](https://spark.apache.org/docs/latest/api/python/) dataframes! :tada:
Use them in all places where Streamlit accepts pandas dataframes, and
they will be automatically converted and evaluated.
"""

"### Automatic conversion"
"First, create a session with Snowpark:"
with st.echo():
    session = Session.builder.configs(st.secrets.snowflake).create()

"Download 100 rows of a sample dataset:"
with st.echo():
    df = (
        session.table("SNOWFLAKE_SAMPLE_DATA.TPCDS_SF100TCL.CATALOG_PAGE")
        .limit(100)
        .collect()
    )

"Pass `df` straight to `st.dataframe` and it *just works*! No conversion to pandas needed."
with st.echo():
    st.dataframe(df)

"Also works with all other Streamlit commands that accept dataframes, e.g. charts, widgets, or maps:"

with st.echo():
    st.line_chart(df, y="CP_CATALOG_PAGE_SK")

"---"
"""
### Automatic evaluation
That's not all! You can also pass your *unevaluated* dataframe (i.e. no `collect()`) directly
to Streamlit. It will automatically download and convert the data. Large tables get
capped at 10k rows, see the little warning above the table.
"""
with st.echo():
    df = session.table("SNOWFLAKE_SAMPLE_DATA.TPCDS_SF100TCL.CATALOG_PAGE")
    st.dataframe(df)

st.info(
    "Capping at 10k rows is great for quick data exploration! No need to worry about app crashes or excessive cost because your data has millions of rows.",
    icon="💡",
)

"---"

st.caption(
    "[Code for this demo](https://github.com/streamlit/release-demos/blob/master/1.16.0/snowpark-support/streamlit_app.py)"
)


import tool.excel
import db
import pymysql
from tool.update import category as book_cate
data = tool.excel.import_data()

def deal():
    conn = db.database_conn()
    cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)
    # book列表
    books = book_cate()

    books_name = []
    for u in books:
        sql = 'select name from type WHERE code="{}"'.format(u)
        cursor.execute(sql)
        res = cursor.fetchone()
        books_name.append(res['name'])
    print(books_name)

    for item in data.keys():
        clothes = item
        for cate in data[item]:
            category = cate
            book_li=[]

            # 构造sql
            sql = """
                            insert into clothes(clothes,category
                            """

            for i in books:
                sql += "," + "li_" + i

            sql += ") VALUES('{}'"
            for j in range(0, 1 + len(books)):
                sql += "," + "'{}'"
            sql += ")"
            for name in books_name:
                sql_type = "select pron from word WHERE `type`='{}' and word='{}'".format(name,category)
                cursor.execute(sql_type)
                res = cursor.fetchone()
                if res:
                    book_li.append(res['pron'])
                else:
                    book_li.append('')
            print(book_li)
            sql=sql.format(clothes,category,*book_li)

            cursor.execute(sql)
            conn.commit()
    conn.close()