from __future__ import annotations
from typing import Any, Dict, List, Set, Union
from collections import OrderedDict
from .rows import Rows
from .option import Option
from .columns.column import Column
from .index import Index
from .constraint import Constraint
from ..operations.create_table import CreateTable
class Table:
    _columns: Dict[str, Column] = None
    _constraints: Dict[str, Constraint] = None
    _indexes: Dict[str, Constraint] = None
    _indexed_columns = None
    _name: str = ''
    _options: List[Option] = None
    _primary: Index = None
    _rows: Dict[int, List[Union[str, int]]] = None
    _tracking_rows: bool = False
    _schema_errors: List[str] = None
    _schema_warnings: List[str] = None
    _global_errors: List[str] = None
    _global_warnings: List[str] = None

    def __init__(
        self,
        name: str = '',
        columns: List[Column] = None,
        indexes: List[Index] = None,
        constraints: List[Constraint] = None,
        options: List[Option] = None
    ):
        self._name = name
        self._options = [*options] if options else []
        self._primary = None
        self._constraints = {}
        self._columns = {}
        self._indexes = {}
        self._parsing_errors = None
        self._parsing_warnings = None
        self._schema_errors = None
        self._schema_warnings = None
        self._global_errors = []
        self._global_warnings = []
        self._final_global_errors = None
        self._final_global_warnings = None
        self._raw_columns = columns if columns is not None else []
        self._raw_constraints = constraints if constraints is not None else []
        self._raw_indexes = indexes if indexes is not None else []

        if columns is not None:
            for column in columns:
                self.add_column(column)
        if constraints is not None:
            for constraint in constraints:
                self.add_constraint(constraint)
        if indexes is not None:
            for index in indexes:
                self.add_index(index)

    # Errors are no longer tracked live. Instead, I need to finish adjusting the flow so that if any
    # of the error fetchers are called, then the errors are collected from all children and then the
    # proper errors are returned.

    @property
    def global_errors(self):
        """ Public getter.  Returns a list of schema errors

        :returns: A list of schema errors
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._final_global_errors

    @property
    def global_warnings(self):
        """ Public getter.  Returns a list of schema warnings

        :returns: A list of schema warnings
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._final_global_warnings

    @property
    def schema_errors(self):
        """ Public getter.  Returns a list of schema errors

        :returns: A list of schema errors
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._schema_errors

    @property
    def schema_warnings(self):
        """ Public getter.  Returns a list of schema warnings

        :returns: A list of schema warnings
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._schema_warnings

    @property
    def parsing_errors(self):
        """ Public getter.  Returns a list of schema errors

        :returns: A list of schema errors
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._parsing_errors

    @property
    def parsing_warnings(self):
        """ Public getter.  Returns a list of schema warnings

        :returns: A list of schema warnings
        :rtype: list
        """
        if self._schema_errors is None:
            self._find_all_errors()
        return self._parsing_warnings

    @property
    def name(self):
        """ Public getter.  Returns the name of the table.

        :returns: The table name
        :rtype: string
        """

        return self._name

    @property
    def options(self):
        """ Public getter.  Returns a list of table options

        :returns: Table options
        :rtype: list
        """

        return self._options

    def _find_all_errors(self):
        self._schema_errors = []
        self._schema_warnings = []
        self._parsing_errors = []
        self._parsing_warnings = []
        if self._global_errors is not None:
            self._final_global_errors = [*self._global_errors]
        if self._global_warnings is not None:
            self._final_global_warnings = [*self._global_warnings]

        if not self.name:
            self._schema_errors.append("Table missing name")
        if not len(self.columns):
            self._schema_errors.append(f"Table '{self.name}' does not have any columns")

        # start with errors from "children" and append the table name for context
        for type_to_check in [self._columns, self._indexes, self._constraints]:
            for to_check in type_to_check.values():
                for error in to_check.schema_errors:
                    self._schema_errors.append(f"{error} in table '{self.name}'")
                for warning in to_check.schema_warnings:
                    self._schema_warnings.append(f"{warning} in table '{self.name}'")
                for error in to_check.parsing_errors:
                    self._parsing_errors.append(f"{error} in table '{self.name}'")
                for warning in to_check.parsing_warnings:
                    self._parsing_warnings.append(f"{error} in table '{self.name}'")
                if hasattr(to_check, 'global_errors'):
                    for warning in to_check.global_errors:
                        self._final_global_errors.append(f"{error} in table '{self.name}'")
                if hasattr(to_check, 'global_warnings'):
                    for warning in to_check.global_warnings:
                        self._final_global_warnings.append(f"{error} in table '{self.name}'")

        # duplicate names.  This shouldn't really be possible anymore because the add_ methods will
        # throw an exception if we try to add a duplicate, but I'll leave this in just in case.
        for (name, to_check) in {
            'columns': self._columns,
            'constraints': self._constraints,
            'indexes': self._indexes
        }.items():
            if len(to_check) == len(getattr(self, name)):
                continue
            label = name.rstrip('es').rstrip('s')
            found = {}
            duplicates = {}
            for item in to_check:
                if item.name in found:
                    duplicates[item.name] = True
                    continue
                found[item.name] = True
            for key in duplicates.keys():
                self._schema_errors.append(f"Duplicate {label} name found in table '{self.name}': '{key}'")

        # more than one primary key
        primaries = list(filter(lambda index: index.is_primary(), self._indexes.values()))
        if len(primaries) > 1:
            self._schema_errors.append(f"Table '{self.name}' has more than one PRIMARY index")

        # auto increment checks
        if self._columns:
            auto_increment = list(filter(lambda column: column.auto_increment, self._columns.values()))
            if len(auto_increment):
                if len(auto_increment) > 1:
                    self._schema_errors.append(f"Table '{self.name}' has more than one AUTO_INCREMENT column")
                elif not primaries:
                    self._schema_errors.append(
                        f"Table '{self.name}' has an AUTO_INCREMENT column but is missing the PRIMARY index"
                    )
                elif primaries[0].columns[0] != auto_increment[0].name:
                    self.schema_errors.append(
                        "Mismatched indexes in table '%s': column '%s' is the AUTO_INCREMENT column but '%s' is the PRIMARY index column"
                        % (self.name, auto_increment[0].name, primaries[0].columns[0])
                    )

        # indexes on non-existent columns
        for index in self._indexes.values():
            for column in index.columns:
                if not column in self.columns:
                    self._schema_errors.append(
                        f"Table '{self.name}' has index '{index.name}' that references non-existent column '{column}'"
                    )

        # we don't bother checking the constraints to see if they are valid because these are
        # best checked at the database level (since, by definition, foreign key constraints are *typically*
        # against other tables, not within a single table.

    def mark_tracking_rows(self):
        """ Marks the table as having had its rows read, for bookeeping purposes

        The reason this is here is because we keep track of whether or not we are "tracking" rows
        for the current table.  This is for bookeeping purposes, largely as a safe-guard attempt
        to try to identify any more subtle bugs that might creep in.  Normally self._tracking_rows
        gets set to True when we add rows to a table, but if a table is empty then rows will never
        get added, and instead this method must be called to mark the rows as "tracked".
        """
        self._tracking_rows = True
        if self._rows is None:
            self._rows = OrderedDict()

    @property
    def tracking_rows(self) -> bool:
        """ Public getter.  Returns True/False to denote whether or not this table is tracking row records

        To be clear on the distinction: just about any table might have rows.  However,
        that doesn't mean that the mygration system should be syncing rows for that table.
        self.tracking_rows == True denotes that the mygration system thinks that we
        should probably be syncing rows for this table.

        :returns: Whether or not the mygration system is tracking rows on the table
        """
        return self._tracking_rows

    @property
    def columns(self) -> Dict[str, Column]:
        """ Public getter.  Returns an ordered dictionary of table columns """
        return self._columns

    @property
    def indexes(self) -> Dict[str, Index]:
        """ Public getter.  Returns an ordered dictionary of table indexes """
        return self._indexes

    @property
    def constraints(self) -> Dict[str, Constraint]:
        """ Public getter.  Returns an ordered dictionary of table constraints """
        return self._constraints

    @property
    def primary(self) -> Index:
        """ Public getter.  Returns the index object for the primary key """
        return self._primary

    @property
    def rows(self) -> Dict[int, List[Union[str, int]]]:
        """ Public getter.  Returns an ordered dictionary with row data by id """
        return None if self._rows is None else self._rows

    def add_rows(self, rows: Rows) -> Union[str, bool]:
        """ Adds rows to the table

        The rows object has some flexibility in terms of columns: it doesn't just
        assume that a value is provided for every column in the table.  Rather,
        there can be a list of columns and only those columns have values (which
        supports the equivalent of MySQL INSERT queries which only specify values
        for some columns).

        Rows with errors will not be processed

        :returns: An error string if an error is encountered, otherwise True/False
        """
        self._schema_errors = None
        self._schema_warnings = None
        if not isinstance(rows, Rows):
            raise ValueError(
                f"Only objects of class mygrations.core.definitions.rows can be added as rows to a table.  Instead I received an object of class '{rows.__class__.__name__}'"
            )

        if rows.parsing_errors:
            self._global_errors.extend(rows.parsing_errors)
        if rows.parsing_warnings:
            self._global_warnings.extend(rows.parsing_warnings)

        self._tracking_rows = True
        if self._rows is None:
            self._rows = OrderedDict()

        # the rows object may have a list of columns.  If not use our own list of columns
        # remember that self._columns is an OrderedDict so converting its keys to a list
        # actually preserves the order (which is a requirement for us)
        columns = rows.columns if rows.num_explicit_columns else list(self._columns.keys())
        if 'id' not in columns:
            self._global_errors.append(
                "A column named 'id' is required to manage rows in the table, but the id column is missing in the rows for table %s"
                % (self._name, )
            )
            return
        id_index = columns.index('id')

        for values in rows.raw_rows:
            # rows without explicit columns must be checked for matching columns
            if not rows.num_explicit_columns and len(values) != len(columns):
                self._global_errors.append(
                    'Insert values has wrong number of values for table %s and row %s' % (self._name, values)
                )
                continue

            # we need to know the id of this record
            row_id = str(values[id_index])
            if not row_id:
                self._global_errors.append(
                    'Row is missing a value for the id column for table %s and row %s' % (self._name, values)
                )
                continue

            if row_id in self._rows:
                self._global_errors.append('Duplicate row id found for table %s and row %s' % (self.name, values))
                continue

            if not row_id:
                self._global_errors.append('Invalid row id of %s found for table %s' % (row_id, self.name))
                continue

            self._rows[row_id] = OrderedDict(zip(columns, values))
            self._rows[row_id]['id'] = row_id

        return True

    def add_raw_row(self, row: Dict[str, Union[str, int]]) -> Union[str, bool]:
        """ Adds a row into the table as a dictionary instead of a row object

        A bit of repetition here.  This is similar to what happens inside the main
        loop of self.add_rows, but different enough that I'm not trying to combine
        them right this second.

        :returns: An error string if an error is encountered, otherwise True/False
        """
        self._schema_errors = None
        self._schema_warnings = None
        self._tracking_rows = True
        if self._rows is None:
            self._rows = OrderedDict()

        row_id = str(row.get('id'))
        if not row_id:
            raise ValueError("Cannot manage records without an 'id' column and value")
        if row_id in self._rows:
            return 'Duplicate row id found for table %s and row %s' % (self._name, row)

        # make sure we have a value for every column in the row, and build an OrderedDict
        converted_row = OrderedDict()
        for column in self._columns.keys():
            if column in row:
                converted_row[column] = row[column]
            else:
                converted_row[column] = self._columns[column].default
        converted_row['id'] = row_id

        self._rows[row_id] = converted_row

    def column_before(self, column_name: str) -> Union[str, bool]:
        """ Returns the name of the column that comes before a given row.

        Returns true if the column is at the beginning of the table

        :returns: The name of the column before the given column, or True if at the beginning
        """
        # this works because we used an OrderedDict
        columns = [col for col in self.columns.keys()]

        if not column_name in columns:
            raise ValueError(
                "Cannot return column before %s because %s does not exist in table %s" %
                (column_name, column_name, self.name)
            )

        index = columns.index(column_name)
        if index == 0:
            return True

        return columns[index - 1]

    def column_is_indexed(self, column: Union[str, Column]) -> bool:
        """ Returns True/False to denote whether or not the column has a useable index """
        if type(column) != str:
            column = column.name
        if column not in self._columns:
            return False

        # best way to do this is with a set.  We'll keep a record of all indexed columns
        # the column is indexed if an index has that column in the first position
        if self._indexed_columns is None:
            self._indexed_columns = set([index.columns[0] for index in self._indexes.values()])
        return column in self._indexed_columns

    def __str__(self) -> str:
        return str(self.create())

    def create(self, nice=False):
        """ Returns a create table operation that can create this table

        :param nice: Whether or not to return a nicely formatted CREATE TABLE command
        :returns: A create table operation
        """
        return CreateTable(self, nice)

    def nice(self) -> str:
        return str(self.create(True))

    def add_column(self, column: Column, position=False):
        """ Adds a column to the table

        The value of position matches self.position from mygrations.formats.mysql.mygration.operations.add_column

        :param column: The column to add
        :param position: Where to put the column
        """
        self._schema_errors = None
        self._schema_warnings = None
        if column.name in self._columns:
            raise ValueError("Cannot add column %s because %s already exists" % (column.name, column.name))

        # putting it at the end is easy
        if not position:
            self._columns[column.name] = column
            return None

        # beginning is also easy
        if position == True:
            self._columns[column.name] = column
            self._columns.move_to_end(column.name, last=False)
            return None

        # now it is tricky
        found = False
        new_columns = OrderedDict()
        for key, value in self._columns.items():
            new_columns[key] = value
            if key == position:
                new_columns[column.name] = column
                found = True

        if not found:
            raise ValueError(
                "Cannot add column %s after %s because %s does not exist" % (column.name, position, position)
            )

        self._columns = new_columns
        return None

    def remove_column(self, column: Union[str, Column]):
        """ Removes a column from the table """
        self._schema_errors = None
        self._schema_warnings = None
        column_name = column if type(column) == str else column.name
        if not column_name in self._columns:
            raise ValueError("Cannot remove column %s because column %s does not exist" % (column_name, column_name))
        self._columns.pop(column_name, None)

    def change_column(self, new_column: Column):
        """ Changes a column.  This does not currently support renaming columns """
        self._schema_errors = None
        self._schema_warnings = None
        if not new_column.name in self._columns:
            raise ValueError(
                "Cannot modify column %s because column %s does not exist" % (new_column.name, new_column.name)
            )
        self._columns[new_column.name] = new_column

    def add_index(self, index: Index):
        """ Adds an index to the table """
        self._schema_errors = None
        self._schema_warnings = None
        if index.name in self._indexes:
            raise ValueError("Cannot add index %s because index %s already exists" % (index.name, index.name))
        if index.index_type == 'PRIMARY':
            self._primary = index
        self._indexes[index.name] = index

        if self._indexed_columns is not None:
            self._indexed_columns.add(index.columns[0])

    def remove_index(self, index: Union[str, Index]):
        """ Removes an index from the table """
        self._schema_errors = None
        self._schema_warnings = None
        index_name = index if type(index) == str else index.name
        if index_name not in self._indexes:
            raise ValueError("Cannot remove index %s because index %s does not exist" % (index_name, index_name))

        indexed_column = self._indexes[index_name].columns[0]
        self._indexes.pop(index_name, None)
        if self._indexed_columns is not None:
            self._indexed_columns.discard(indexed_column)

    def change_index(self, new_index: Index):
        """ Changes an index.  This does not currently support renaming """
        self._schema_errors = None
        self._schema_warnings = None
        if not new_index.name in self._indexes:
            raise ValueError(
                "Cannot modify index %s because index %s does not exist" % (new_index.name, new_index.name)
            )

        if self._indexed_columns is not None:
            self._indexed_columns.discard(self._indexes[new_index.name].columns[0])
        self._indexes[new_index.name] = new_index

        if self._indexed_columns is not None:
            self._indexed_columns.add(new_index.columns[0])

    def add_constraint(self, constraint: Constraint):
        """ Adds a constraint to the table """
        self._schema_errors = None
        self._schema_warnings = None
        if constraint.name in self._constraints:
            raise ValueError(
                "Cannot add constraint %s because constraint %s already exists" % (constraint.name, constraint.name)
            )
        self._constraints[constraint.name] = constraint

    def remove_constraint(self, constraint: Union[str, Constraint]):
        """ Removes an constraint from the table """
        self._schema_errors = None
        self._schema_warnings = None
        if type(constraint) != str:
            constraint = constraint.name
        if constraint not in self._constraints:
            raise ValueError(
                "Cannot remove constraint %s because constraint %s does not exist" % (constraint, constraint)
            )
        self._constraints.pop(constraint, None)

    def change_constraint(self, new_constraint):
        """ Changes a constraint. This does not currently support renaming. """
        self._schema_errors = None
        self._schema_warnings = None
        if not new_constraint.name in self._constraints:
            raise ValueError(
                "Cannot modify constraint %s because constraint %s does not exist" %
                (new_constraint.name, new_constraint.name)
            )
        self._constraints[new_constraint.name] = new_constraint

    def _loose_equal(self, val1: Union[str, int], val2: Union[str, int]) -> bool:
        """ Performs a looser comparison, as values might have different types depending on whether they came out of a database or file

        Returns true if the two values are equal, even if one is a string and the other an int.
        """
        # if we don't have a type mistmatch then this is easy
        if type(val1) == type(val2):
            return val1 == val2

        # otherwise see if we can cheat and just convert to strings
        return str(val1) == str(val2)

    def to(self, comparison_table, split_operations=False):
        """ Compares two tables to eachother and returns a list of operations which can bring the structure of the second in line with the first

        In other words, this pseudo code will make table have the same structure as comparison_table

        for operation in table.to( comparison_table ):
            table.apply( operation )

        if split_operations is True then a dict of migration operations will be returned to separate
        foreign key operations from everything else.  The dict will have up to three keys: [ 'removed_fks', 'fks', 'kitchen_sink' ]

        'fks' contains the alter statement operation needed to add/change/remove foreign keys
        'kitchen_sink' contains everything else

        If that division of labor seems arbitrary, it isn't: it is separated out that
        way due to the needs of the overall algorithm.

        If split_operations is false then a single alter table operation will be returned that encompasses all changes

        :param comparison_table: A table to find differences with
        :param split_operations: Whether to combine all operations in one alter table or separate them
        :type comparison_table: mygrations.formats.mysql.definitions.table
        :type split_operations: bool
        :returns: A list of operations to apply to table
        :rtype: list[mygrations.formats.mysql.mygrations.operations.*] | dict
        """
        raise NotImplementedError()

    def to_rows(self, from_table=None):
        """ Compares two tables to eachother and returns a list of operations which can bring the rows of this table in line with the other

        It's important to note (and probably important to think through and change eventually)
        that this method has the opposite meaning of `mygrations.formats.mysql.definitions.table.to()`
        That method is called on the `from` table and operations on the (required) `to` table.
        This method is called on the `to` table and can (optionally) be passed in a `from` table.

        :param from_table: A table to find differences with (or None)
        :type from_table: mygrations.formats.mysql.definitions.table
        :returns: A list of operations to apply to table
        :rtype: list[mygrations.formats.mysql.mygrations.operations.*]
        """
        raise NotImplementedError()

    def _differences(self, from_list: Dict[str, Any], to_list: Dict[str, Any]):
        """
        Calculates the difference between two OrderedDicts.

        https://codereview.stackexchange.com/a/176303/140581

        :param from_list: OrderedDict
        :param to_list: OrderedDict
        :return: (added, removed, overlap)
        """

        return ([key for key in to_list if key not in from_list], [key for key in from_list if key not in to_list],
                [key for key in from_list if key in to_list])

    def apply_operation(self, operation):
        """
        Applies an operation to the table

        :param operation: The operation to apply
        :type operation: mygrations.formats.mysql.mygration.operations.*
        """
        operation.apply_to_table(self)


# USAGE
# To read and write back out to video:
# python people_counter.py --prototxt mobilenet_ssd/MobileNetSSD_deploy.prototxt \
#	--model mobilenet_ssd/MobileNetSSD_deploy.caffemodel --input videos/example_01.mp4 \
#	--output output/output_01.avi
#
# To read from webcam and write back out to disk:
# python people_counter.py --prototxt mobilenet_ssd/MobileNetSSD_deploy.prototxt \
#	--model mobilenet_ssd/MobileNetSSD_deploy.caffemodel \
#	--output output/webcam_output.avi

# import the necessary packages
from pyimagesearch.centroidtracker import CentroidTracker
from pyimagesearch.trackableobject import TrackableObject
from imutils.video import VideoStream
from imutils.video import FPS
import numpy as np
import argparse
import imutils
import time
import dlib
import cv2

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-p", "--prototxt", required=True,
	help="path to Caffe 'deploy' prototxt file")
ap.add_argument("-m", "--model", required=True,
	help="path to Caffe pre-trained model")
ap.add_argument("-i", "--input", type=str,
	help="path to optional input video file")
ap.add_argument("-o", "--output", type=str,
	help="path to optional output video file")
ap.add_argument("-c", "--confidence", type=float, default=0.2,
	help="minimum probability to filter weak detections")
ap.add_argument("-s", "--skip-frames", type=int, default=10,
	help="# of skip frames between detections")
args = vars(ap.parse_args())

# initialize the list of class labels MobileNet SSD was trained to
# detect
CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
	"bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
	"dog", "horse", "motorbike", "person", "pottedplant", "sheep",
	"sofa", "train", "tvmonitor"]

# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(args["prototxt"], args["model"])

# if a video path was not supplied, grab a reference to the webcam
if not args.get("input", False):
	print("[INFO] starting video stream...")
	vs = VideoStream(src=0).start()
	time.sleep(2.0)

# otherwise, grab a reference to the video file
else:
	print("[INFO] opening video file...")
	vs = cv2.VideoCapture(args["input"])

# initialize the video writer (we'll instantiate later if need be)
writer = None

# initialize the frame dimensions (we'll set them as soon as we read
# the first frame from the video)
W = None
H = None

# instantiate our centroid tracker, then initialize a list to store
# each of our dlib correlation trackers, followed by a dictionary to
# map each unique object ID to a TrackableObject
ct = CentroidTracker(maxDisappeared=40, maxDistance=50)
trackers_c = []
trackers_p = []
trackers = []
labels = [] #added labels list initialization

trackableObjects_c = {}
trackableObjects_p = {}
rect_c =[]
rect_p =[]
boxes_c = []
boxes_p =[]
counter = 0

# initialize the total number of frames processed thus far, along
# with the total number of objects that have moved either up or down
totalFrames = 0
totalDown = 0
totalUp = 0

# start the frames per second throughput estimator
fps = FPS().start()

# loop over frames from the video stream
while True:
	# grab the next frame and handle if we are reading from either
	# VideoCapture or VideoStream
	frame = vs.read()
	frame = frame[1] if args.get("input", False) else frame

	# if we are viewing a video and we did not grab a frame then we
	# have reached the end of the video
	if args["input"] is not None and frame is None:
		break

	# resize the frame to have a maximum width of 500 pixels (the
	# less data we have, the faster we can process it), then convert
	# the frame from BGR to RGB for dlib
	frame = imutils.resize(frame, width=500)
	rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

	# if the frame dimensions are empty, set them
	if W is None or H is None:
		(H, W) = frame.shape[:2]

	# if we are supposed to be writing a video to disk, initialize
	# the writer
	if args["output"] is not None and writer is None:
		fourcc = cv2.VideoWriter_fourcc(*"MJPG")
		writer = cv2.VideoWriter(args["output"], fourcc, 30,
			(W, H), True)

	# initialize the current status along with our list of bounding
	# box rectangles returned by either (1) our object detector or
	# (2) the correlation trackers
	status = "Waiting"
	rects_p = []
	rects_c = []

	# check to see if we should run a more computationally expensive
	# object detection method to aid our tracker
	if totalFrames % args["skip_frames"] == 0:
		# set the status and initialize our new set of object trackers
		status = "Detecting"
		trackers_c = []
		trackers_p = []

		# convert the frame to a blob and pass the blob through the
		# network and obtain the detections
		blob = cv2.dnn.blobFromImage(frame.copy(), 0.007843, (W, H), 127.5)
		net.setInput(blob)
		detections = net.forward()

		# loop over the detections
		for i in np.arange(0, detections.shape[2]):
			# extract the confidence (i.e., probability) associated
			# with the prediction
			confidence = detections[0, 0, i, 2]

			# filter out weak detections by requiring a minimum
			# confidence
			if confidence > args["confidence"]:
				# extract the index of the class label from the
				# detections list
				idx = int(detections[0, 0, i, 1])
				label = CLASSES[idx] #new addition

				# if the class label is not a person, ignore it #new addition
				if (CLASSES[idx] != "person") & (CLASSES[idx]!= "chair") & (CLASSES[idx]!= "sofa"):
					continue

				if (CLASSES[idx] == "person"):
					# compute the (x, y)-coordinates of the bounding box
					# for the object
					box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
					(startXp, startYp, endXp, endYp) = box.astype("int")

					# construct a dlib rectangle object from the bounding
					# box coordinates and then start the dlib correlation
					# tracker
					tracker = dlib.correlation_tracker()
					rect = dlib.rectangle(int(startXp), int(startYp), int(endXp), int(endYp))
					tracker.start_track(rgb, rect)
					rect_p = box
					boxes_p.append(rect_p)

					# add the tracker to our list of trackers so we can
					# utilize it during skip frames
					trackers_p.append(tracker)
					labels.append(label) #new addition

					# new addition - grab the corresponding class label for the detection
					# and draw the bounding box
					cv2.rectangle(frame, (startXp, startYp), (endXp, endYp),
						(0, 255, 0), 2)
					if (label == 'person'):
						cv2.putText(frame, label, (startXp, startYp - 15),
							cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 2)

				if (CLASSES[idx] == "chair") | (CLASSES[idx] == "sofa"):
					# compute the (x, y)-coordinates of the bounding box
					# for the object
					box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
					(startXc, startYc, endXc, endYc) = box.astype("int")

					# construct a dlib rectangle object from the bounding
					# box coordinates and then start the dlib correlation
					# tracker
					tracker = dlib.correlation_tracker()
					rect = dlib.rectangle(int(startXc), int(startYc), int(endXc), int(endYc))
					tracker.start_track(rgb, rect)
					rect_c = box
					boxes_c.append(rect_c)

					# add the tracker to our list of trackers so we can
					# utilize it during skip frames
					trackers_c.append(tracker)
					labels.append(label) #new addition

					# new addition - grab the corresponding class label for the detection
					# and draw the bounding box
					cv2.rectangle(frame, (startXc, startYc), (endXc, endYc),
						(0, 255, 0), 2)
					if (label == 'chair') | (label == 'sofa'):
						cv2.putText(frame, label, (startXc, startYc - 15),
							cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 2)

			#for i in enumerate(boxes_p, boxes_c):
					#if ((int(boxes_p[i][0]) ==  int(boxes_c[i][0])) | (int(boxes_p[i][0]) ==  int(boxes_c[i][0]+1)) |
						#(int(boxes_p[i][0]) ==  int(boxes_c[i][0]-1))) & ((int(boxes_p[i][3]) ==  int(boxes_c[i][3])) |
						#(int(boxes_p[i][3]) ==  int(boxes_c[i][3]+1)) | (int(boxes_p[i][3]) ==  int(boxes_c[i][3]-1))):
							#counter += 1
				#print(counter)

	# otherwise, we should utilize our object *trackers* rather than
	# object *detectors* to obtain a higher frame processing throughput
	else:
		# loop over the trackers
		for trackerp in trackers_p:
			# set the status of our system to be 'tracking' rather
			# than 'waiting' or 'detecting'
			status = "Tracking_p"

			# update the tracker and grab the updated position
			trackerp.update(rgb)
			pos = trackerp.get_position()

			# unpack the position object
			startXp = int(pos.left())
			startYp = int(pos.top())
			endXp = int(pos.right())
			endYp = int(pos.bottom())

			# add the bounding box coordinates to the rectangles list
			rects_p.append((startXp, startYp, endXp, endYp))

            # new addition- draw the bounding box from the correlation object tracker
			cv2.rectangle(frame, (startXp, startYp), (endXp, endYp),
				(0, 255, 0), 2)
			if (label == 'person'):
				cv2.putText(frame, label, (startXp, startYp - 15),
					cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 2)

		# loop over the trackers
		for trackerc in trackers_c:
			# set the status of our system to be 'tracking' rather
			# than 'waiting' or 'detecting'
			status = "Tracking"

			# update the tracker and grab the updated position
			trackerc.update(rgb)
			pos = trackerc.get_position()

			# unpack the position object
			startXc = int(pos.left())
			startYc = int(pos.top())
			endXc = int(pos.right())
			endYc = int(pos.bottom())

			# add the bounding box coordinates to the rectangles list
			rects_c.append((startXc, startYc, endXc, endYc))

            # new addition- draw the bounding box from the correlation object tracker
			cv2.rectangle(frame, (startXc, startYc), (endXc, endYc),
				(0, 255, 0), 2)
			if (label == 'chair'):
				cv2.putText(frame, label, (startXc, startYc - 15),
					cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 2)

	# draw a horizontal line in the center of the frame -- once an
	# object crosses this line we will determine whether they were
	# moving 'up' or 'down'
	cv2.line(frame, (0, (H-100)//2),((W+500)//2, H-120), (0, 255, 255), 2)

	# use the centroid tracker to associate the (1) old object
	# centroids with (2) the newly computed object centroids
	objects_p = ct.update(rects_p)
	objects_c = ct.update(rects_c)

	# loop over the tracked objects
	for (objectID, centroid) in objects_p.items():
	# check to see if a trackable object exists for the current
	# object ID
		to_p = trackableObjects_p.get(objectID, None)

	# if there is no existing trackable object, create one
		if to_p is None:
			to_p = TrackableObject(objectID, centroid)

	# otherwise, there is a trackable object so we can utilize it
	# to determine direction
		else:
		# the difference between the y-coordinate of the *current*
		# centroid and the mean of *previous* centroids will tell
		# us in which direction the object is moving (negative for
		# 'up' and positive for 'down')
			y = [c[1] for c in to_p.centroids]
			direction = centroid[1] - np.mean(y)
			to_p.centroids.append(centroid)

		# check to see if the object has been counted or not
		#if not to_p.counted:
			# if the direction is negative (indicating the object
			# is moving up) AND the centroid is above the center
				# line, count the object
			#if direction < 0 and centroid[1] < (H-100) // 2:
				#totalUp += 1
				#to_p.counted = True

			# if the direction is positive (indicating the object
			# is moving down) AND the centroid is below the
			# center line, count the object
			#elif direction > 0 and centroid[1] > (H-100) // 2:
				#totalDown += 1
				#to_p.counted = True

		# store the trackable object in our dictionary
		trackableObjects_p[objectID] = to_p

	# loop over the tracked objects
	for (objectID, centroid) in objects_c.items():
		# check to see if a trackable object exists for the current
		# object ID
		to_c = trackableObjects_c.get(objectID, None)

		# if there is no existing trackable object, create one
		if to_c is None:
			to_c = TrackableObject(objectID, centroid)

		# otherwise, there is a trackable object so we can utilize it
		# to determine direction
		else:
			# the difference between the y-coordinate of the *current*
			# centroid and the mean of *previous* centroids will tell
			# us in which direction the object is moving (negative for
			# 'up' and positive for 'down')
			y = [c[1] for c in to_c.centroids]
			direction = centroid[1] - np.mean(y)
			to_c.centroids.append(centroid)

			# check to see if the object has been counted or not
			#if not to_c.counted:
				# if the direction is negative (indicating the object
				# is moving up) AND the centroid is above the center
				# line, count the object
				#if direction < 0 and centroid[1] < (H-100) // 2:
					#totalUp += 1
					#to_c.counted = True

				# if the direction is positive (indicating the object
				# is moving down) AND the centroid is below the
				# center line, count the object
				#elif direction > 0 and centroid[1] > (H-100) // 2:
					#totalDown += 1
					#to_c.counted = True

		# store the trackable object in our dictionary
		trackableObjects_c[objectID] = to_c

	#clist = list(objects_c.items())
	#clist_cent = []
	#for i in range(len(clist)):
		#clist_cent.append(clist[1])

	#cv2.circle(frame, (W-250,H-150), 4, (0, 255, 0), -1)
	#cv2.circle(frame, (W-250,H-100), 4, (0, 255, 0), -1)
	#cv2.circle(frame, (W-250,H-200), 4, (0, 255, 0), -1)
	#cv2.circle(frame, (W-250,H-180), 4, (0, 255, 0), -1)
	for objectIDc, centroid_c in objects_c.items():
		#Cx = centroid_c[0]
		#Cy = centroid_c[1]
		#to_c = TrackableObject(objectIDc, centroid_c)
		for objectIDp, centroid_p in objects_p.items():
			#to_p = TrackableObject(objectIDp, centroid_p)
			if (centroid_c[0] == centroid_p[0]) & (centroid_c[1] == centroid_p[1]) :
				if (to_p.counted == False) & (to_c.counted == False) & (centroid_p[1] < (H-150)):
					totalUp += 1
					to_p.counted = True
					to_c.counted = True
				if (to_p.counted == False) & (to_c.counted == False) & (centroid_p[1] > (H-180)):
					totalDown += 1
					to_p.counted = True
					to_c.counted = True

		# draw both the ID of the object and the centroid of the
		# object on the output frame
		#text = "ID {}".format(objectID)
		#cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10),
			#cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
		#cv2.circle(frame, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
		#cv2.rectangle(frame, (centroid[0]-20, centroid[1]-40), (centroid[0]+20, centroid[1]+40), (0, 255, 0), 2) #added line

	# construct a tuple of information we will be displaying on the
	# frame
	info = [
		("Sunlit", totalUp),
		("Shade", totalDown),
		("Status", status),
	]

	# loop over the info tuples and draw them on our frame
	for (i, (k, v)) in enumerate(info):
		text = "{}: {}".format(k, v)
		cv2.putText(frame, text, (10, H - ((i * 20) + 20)),
			cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)

	# check to see if we should write the frame to disk
	if writer is not None:
		writer.write(frame)

	# show the output frame
	cv2.imshow("Frame", frame)
	key = cv2.waitKey(1) & 0xFF

	# if the `q` key was pressed, break from the loop
	if key == ord("q"):
		break

	# increment the total number of frames processed thus far and
	# then update the FPS counter
	totalFrames += 1
	fps.update()

# stop the timer and display FPS information
fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

# check to see if we need to release the video writer pointer
if writer is not None:
	writer.release()

# if we are not using a video file, stop the camera video stream
if not args.get("input", False):
	vs.stop()

# otherwise, release the video file pointer
else:
	vs.release()

# close any open windows
cv2.destroyAllWindows()

Object Tracking Python Script/people_counter4.py


import os
import imageio
import argparse
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

from natsort import natsorted
from PIL import Image, ImageDraw, ImageFont

def plot_generated(generated_imgs, dim=(1, 10), figsize=(12, 2), save_name=None):
  plt.figure(figsize=figsize)
  for i in range(generated_imgs.shape[0]):
    plt.subplot(dim[0], dim[1], i+1)
    plt.imshow(generated_imgs[i], interpolation='nearest', cmap='gray_r')
    plt.axis('off')
  plt.tight_layout()
  # plt.draw()
  # plt.pause(0.01)
  if save_name is None:
    plt.show()
  else:
    plt.savefig(save_name)
    plt.close()

def create_gif(img_folder, save_gif_name=None, fps=5):
  images = [f'{img_folder}{img}' for img in natsorted(os.listdir(img_folder))]

  gif = []
  for image in images:
      gif.append(imageio.imread(image))

  if save_gif_name is None:
    save_gif_namef = f'{img_folder[:-1]}_animated.gif'

  imageio.mimsave(save_gif_namef, gif, fps=fps)

def read_subset_classif_results(log_file):
  with open(log_file, 'r') as f:
    data = f.read().splitlines()

  starts_ends = [i for i, el in enumerate(data) if 'START' in el or 'END' in el]
  starts_ends = starts_ends[:len(starts_ends) - len(starts_ends) % 2]

  results = {'epoch': [], 'f1': [], 'label': []}
  percents = []
  for i, idx in enumerate(range(0, len(starts_ends), 2)):
    # if i == 0:
    #   continue
    tmp = data[starts_ends[idx]:starts_ends[idx+1]]
    epochs, f1s = zip(*[(int(l.split(' | ')[0].split('Epoch ')[-1]), float(l.split(' = ')[-1])) for l in tmp if ' | f1 = ' in l])
    results['epoch'] += epochs
    results['f1'] += f1s
    results['label'] += [i] * len(epochs)
    # percents.append(tmp[0].split(' = ')[-1])
    percents.append(str(int(float(tmp[0].split(' = ')[-1]) * 6000)))
    # print(f'Percent = {percents[-1]} | max f1 = {max(f1s)}')
    print(f'n_examples_per_class = {percents[-1]} | max f1 = {max(f1s)}')

  lp = sns.lineplot(x='epoch', y='f1', hue='label', data=pd.DataFrame.from_dict(results),
                    legend=False, palette=sns.color_palette()[:len(percents)])
  # lp.set(yscale="log")
  plt.legend(title='percent', loc='lower right', labels=percents)
  plt.show()

def read_ssdcgan_logs(folder='tmp_data/', root_name='_tmp_mnist_gan_ssdcgan_percent{}_logs.txt'):
  for percent in [0.002, 0.004, 0.009, 0.017, 0.084, 0.17, 0.34, 0.67, 1.]:
    fname = os.path.join(folder, root_name.format(percent)).replace('0.', '0')
    if not os.path.isfile(fname):
      continue

    with open(fname, 'r') as f:
      data = f.read().splitlines()

    f1s = [float(l.split(' = ')[-1]) for l in data if 'Saving model with' in l]
    n_examples = int(percent * 6000)
    print(f'n_examples = {n_examples} | f1 = {np.max(f1s)}')

def plot_comp_imgs(folder='tmp_data/', fname='imgs_generated_epoch180.png'):
  dcgan, cdcgan, acdcgan = 'generated_dcgan_imgs/', 'generated_cdcgan_imgs/', 'generated_acdcgan_imgs/'

  img_dcgan = Image.open(os.path.join(folder, dcgan, fname))
  img_cdcgan = Image.open(os.path.join(folder, cdcgan, fname))
  img_acdcgan = Image.open(os.path.join(folder, acdcgan, fname))

  font = ImageFont.truetype('open-sans/OpenSans-Regular.ttf', 20)

  for text, tmp_img in zip(['DC-GAN', 'Conditional DC-GAN', 'AC-DC-GAN'], [img_dcgan, img_cdcgan, img_acdcgan]):
    draw = ImageDraw.Draw(tmp_img)
    draw.text((5, 5), text, font=font, fill=(0, 0, 0))

  img = Image.new('L', (img_dcgan.width, img_dcgan.height + img_cdcgan.height + img_acdcgan.height))

  img.paste(img_dcgan, (0, 0))
  img.paste(img_cdcgan, (0, img_dcgan.height))
  img.paste(img_acdcgan, (0, 2 * img_dcgan.height))

  img.show()

if __name__ == '__main__':
  argparser = argparse.ArgumentParser(prog='plotter.py', description='')
  argparser.add_argument('--log_file', default='_tmp_classif_exps_mnist_logs.txt', type=str)
  args = argparser.parse_args()

  rep = input('Read subset classification results? (y or n): ')
  if rep == 'y':
    read_subset_classif_results(args.log_file)

  rep = input('Read ssdcgan logs? (y or n): ')
  if rep == 'y':
    read_ssdcgan_logs()

  rep = input('Plot comparison of generated GAN images? (y or n): ')
  if rep == 'y':
    plot_comp_imgs()


"""
Document Scanner
Algo :  Take input from webcom
        Preprocess Image and return as Threshold image
        Find the biggest contour
        Using corner points to get bird eye view

"""

import cv2
import numpy as np

window_width = 600
window_height = 350
webcam = cv2.VideoCapture(0)   # Selecting webcam
webcam.set(3,window_width)   # Adjust Width
webcam.set(4,window_height)   # Adjust Height
webcam.set(10,150)   # Adjust Brightness

def imagePreProcessing(img):
    imgGray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    imgBlur = cv2.GaussianBlur(imgGray,(5,5),1)
    imgCanny = cv2.Canny(imgBlur,100,100)
    kernal = np.ones([5,5])
    imgDilate = cv2.dilate(imgCanny,kernal,iterations=1)
    imgErode = cv2.erode(imgDilate, kernal, iterations=1)
    return imgDilate

def getContours(img):
    biggest_box = np.array([])
    maxArea = 0
    contours,hierarchy = cv2.findContours(img,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    x, y, w, h = 0, 0, 0, 0
    for cnt in contours:
        area = cv2.contourArea(cnt)
        if area>5000:
            # cv2.drawContours(imgCountour, cnt, -1, (255, 0, 0),3)
            parameter = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt, 0.02 * parameter, True)  ## Find approximation of our corner points
            # print(len(approx))
            if len(approx) == 4 and area > maxArea:
                biggest_box = approx
                maxArea = area
    cv2.drawContours(imgCountour, biggest_box, -1, (255, 0, 0),20)
    return biggest_box

def reorder(points):
    points = points.reshape((4,2))
    newPoints = np.zeros((4,1,2),np.int32)

    add = points.sum(1)
    newPoints[0] = points[np.argmin(add)]    ## setting point [0,0]
    newPoints[3] = points[np.argmax(add)]   ## setting point [width,height]

    diff = np.diff(points, axis=1)
    newPoints[1] = points[np.argmin(diff)]  ## setting point [width,0]
    newPoints[2] = points[np.argmax(diff)]  ## setting point [0,height]
    return newPoints

def wrap(img,biggestContourPoint):
    biggestContourPoint = reorder(biggestContourPoint)
    width = window_width
    height = window_height
    pts1 = np.float32(biggestContourPoint)
    pts2 = np.float32([[0,0],[width,0],[0,height],[width,height]])
    matrix = cv2.getPerspectiveTransform(pts1, pts2)
    imgOutput = cv2.warpPerspective(img, matrix, (width, height))
    return imgOutput

def stackImages(scale,imgArray):
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shape[0]
    if rowsAvailable:
        for x in range ( 0, rows):
            for y in range(0, cols):
                if imgArray[x][y].shape[:2] == imgArray[0][0].shape [:2]:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (0, 0), None, scale, scale)
                else:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0].shape[1], imgArray[0][0].shape[0]), None, scale, scale)
                if len(imgArray[x][y].shape) == 2: imgArray[x][y]= cv2.cvtColor( imgArray[x][y], cv2.COLOR_GRAY2BGR)
        imageBlank = np.zeros((height, width, 3), np.uint8)
        hor = [imageBlank]*rows
        hor_con = [imageBlank]*rows
        for x in range(0, rows):
            hor[x] = np.hstack(imgArray[x])
        ver = np.vstack(hor)
    else:
        for x in range(0, rows):
            if imgArray[x].shape[:2] == imgArray[0].shape[:2]:
                imgArray[x] = cv2.resize(imgArray[x], (0, 0), None, scale, scale)
            else:
                imgArray[x] = cv2.resize(imgArray[x], (imgArray[0].shape[1], imgArray[0].shape[0]), None,scale, scale)
            if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
        hor= np.hstack(imgArray)
        ver = hor
    return ver

while True:
    _,img = webcam.read()
    img = cv2.resize(img, (500, 200))
    imgCountour = img.copy()
    imgThresh = imagePreProcessing(img)
    biggestContour = getContours(imgThresh)
    # print(biggestContour,biggestContour.shape)
    imgStagedArray = []
    if biggestContour.size != 0:
        imgWrapOutput = wrap(img, biggestContour)
        cv2.imshow("Document Scanner", imgWrapOutput)
        imgStagedArray = [[img,imgThresh],[imgCountour,imgWrapOutput]]
    else:
        imgStagedArray = [[img,imgThresh],[img,img]]

    stackedImg = stackImages(0.6, imgStagedArray)
    cv2.imshow("Workflow", stackedImg)

    if cv2.waitKey(1) & 0xFF == ord("q"):
        break


from __future__ import absolute_import

import itertools
import math
import warnings
from typing import List, Iterable, Tuple, Optional

import cv2
import numpy as np

from .torch_utils import image_to_tensor

__all__ = [
    "plot_confusion_matrix",
    "render_figure_to_tensor",
    "hstack_autopad",
    "vstack_autopad",
    "vstack_header",
    "grid_stack",
    "plot_heatmap",
]

def plot_heatmap(
    cm: np.ndarray,
    title: str,
    x_label=None,
    y_label=None,
    x_ticks: List[str] = None,
    y_ticks: List[str] = None,
    format_string=None,
    show_scores=True,
    fontsize=12,
    figsize: Tuple[int, int] = (16, 16),
    fname=None,
    noshow: bool = False,
    cmap=None,
    backend="Agg",
):
    if len(cm.shape) != 2:
        raise ValueError("Heatmap must be a 2-D array")
    import matplotlib

    matplotlib.use(backend)
    import matplotlib.pyplot as plt

    if cmap is None:
        cmap = plt.cm.Oranges

    f = plt.figure(figsize=figsize)
    plt.imshow(cm, interpolation="nearest", cmap=cmap)
    plt.title(title)
    plt.colorbar(fraction=0.046, pad=0.04)

    if x_ticks is not None:
        plt.xticks(np.arange(len(x_ticks)), x_ticks, rotation=45, ha="right")

    if y_ticks is not None:
        plt.yticks(np.arange(len(y_ticks)), y_ticks)

    if format_string is None:
        format_string = ".2f" if np.issubdtype(cm.dtype, np.floating) else "d"

    if show_scores:
        thresh = (cm.max() + cm.min()) / 2.0
        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
            text = format(cm[i, j], format_string) if np.isfinite(cm[i, j]) else "N/A"
            color = "white" if cm[i, j] > thresh else "black"
            plt.text(
                j,
                i,
                text,
                horizontalalignment="center",
                verticalalignment="center_baseline",
                fontsize=fontsize,
                color=color,
            )

    plt.xlabel(x_label)
    plt.ylabel(y_label)

    plt.tight_layout()

    if fname is not None:
        plt.savefig(fname=fname, dpi=200)

    if not noshow:
        plt.show()

    return f

def plot_confusion_matrix(
    cm: np.ndarray,
    class_names: List[str],
    figsize: Tuple[int, int] = (16, 16),
    fontsize: int = 12,
    normalize: bool = False,
    title: str = "Confusion matrix",
    cmap=None,
    fname=None,
    show_scores: bool = True,
    noshow: bool = False,
    backend: str = "Agg",
    format_string: Optional[str] = None,
):
    """
    Render the confusion matrix and return matplotlib's figure with it.
    Normalization can be applied by setting `normalize=True`.

    Args:
        cm: Numpy array of (N,N) shape - confusion matrix array
        class_names: List of [N] names of the classes
        figsize:
        fontsize:
        normalize: Whether to apply normalization for each row of CM
        title: Title of the confusion matrix
        cmap:
        fname: Filename of the rendered confusion matrix
        show_scores: Show scores in each cell
        noshow:
        backend:
        format_string:

    Returns:
        Matplotlib's figure
    """
    import matplotlib

    matplotlib.use(backend)
    import matplotlib.pyplot as plt

    if cmap is None:
        cmap = plt.cm.Oranges

    if normalize:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            cm = cm.astype(np.float32) / cm.sum(axis=1)[:, np.newaxis]
    else:
        accuracy = np.trace(cm) / (float(np.sum(cm)) + 1e-8)
        misclass = 1 - accuracy

    f = plt.figure(figsize=figsize)
    plt.imshow(cm, interpolation="nearest", cmap=cmap)
    plt.title(title)
    plt.colorbar(fraction=0.046, pad=0.04)

    tick_marks = np.arange(len(class_names))
    plt.xticks(tick_marks, class_names, rotation=45, ha="right")
    plt.yticks(tick_marks, class_names)

    if format_string is None:
        format_string = ".3f" if normalize else "d"

    if show_scores:
        thresh = (cm.max() + cm.min()) / 2.0
        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
            text = format(cm[i, j], format_string) if np.isfinite(cm[i, j]) else "N/A"
            color = "white" if cm[i, j] > thresh else "black"
            plt.text(j, i, text, horizontalalignment="center", fontsize=fontsize, color=color)

    plt.ylabel("True label")

    if normalize:
        # We don't show Accuracy & Misclassification scores for normalized CM
        plt.xlabel("Predicted label")
    else:
        plt.xlabel("Predicted label\nAccuracy={:0.4f}; Misclass={:0.4f}".format(accuracy, misclass))

    plt.tight_layout()

    if fname is not None:
        plt.savefig(fname=fname, dpi=200)

    if not noshow:
        plt.show()

    return f

def render_figure_to_tensor(figure):
    import matplotlib

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt

    figure.canvas.draw()

    # string = figure.canvas.tostring_argb()

    image = np.array(figure.canvas.renderer._renderer)
    plt.close(figure)
    del figure

    image = image_to_tensor(image)
    return image

def hstack_autopad(images: Iterable[np.ndarray], pad_value: int = 0) -> np.ndarray:
    """
    Stack images horizontally with automatic padding

    Args:
        images: List of images to stack

    Returns:
        image
    """
    max_height = 0
    for img in images:
        max_height = max(max_height, img.shape[0])

    padded_images = []
    for img in images:
        height = img.shape[0]
        pad_top = 0
        pad_bottom = max_height - height
        pad_left = 0
        pad_right = 0
        img = cv2.copyMakeBorder(img, pad_top, pad_bottom, pad_left, pad_right, cv2.BORDER_CONSTANT, value=pad_value)
        (rows, cols) = img.shape[0:2]
        padded_images.append(img)

    return np.hstack(padded_images)

def vstack_autopad(images: Iterable[np.ndarray], pad_value: int = 0) -> np.ndarray:
    """
    Stack images vertically with automatic padding

    Args:
        images: List of images to stack

    Returns:
        image
    """
    max_width = 0
    for img in images:
        max_width = max(max_width, img.shape[1])

    padded_images = []
    for img in images:
        width = img.shape[1]
        pad_top = 0
        pad_bottom = 0
        pad_left = 0
        pad_right = max_width - width
        img = cv2.copyMakeBorder(img, pad_top, pad_bottom, pad_left, pad_right, cv2.BORDER_CONSTANT, value=pad_value)
        padded_images.append(img)

    return np.vstack(padded_images)

def vstack_header(
    image: np.ndarray,
    title: str,
    bg_color=(35, 41, 40),
    text_color=(242, 248, 248),
    text_thickness: int = 2,
    text_scale=1.5,
) -> np.ndarray:
    (rows, cols) = image.shape[:2]

    title_image = np.zeros((30, cols, 3), dtype=np.uint8)
    title_image[:] = bg_color
    cv2.putText(
        title_image,
        title,
        (10, 24),
        fontFace=cv2.FONT_HERSHEY_PLAIN,
        fontScale=text_scale,
        color=text_color,
        thickness=text_thickness,
        lineType=cv2.LINE_AA,
    )

    return vstack_autopad([title_image, image])

def grid_stack(images: List[np.ndarray], rows: int = None, cols: int = None) -> np.ndarray:
    if rows is None and cols is None:
        rows = int(math.ceil(math.sqrt(len(images))))
        cols = int(math.ceil(len(images) / rows))
    elif rows is None:
        rows = math.ceil(len(images) / cols)
    elif cols is None:
        cols = math.ceil(len(images) / rows)
    else:
        if len(images) > rows * cols:
            raise ValueError("Number of rows * cols must be greater than number of images")

    image_rows = []
    for r in range(rows):
        image_rows.append(hstack_autopad(images[r * cols : (r + 1) * cols]))

    return vstack_autopad(image_rows)


# -*- coding: utf-8 -*-

from django.http import JsonResponse
from django.db.models import Count, F, Case, When, Value, IntegerField

from what.models import Composer, Instrument
from api.shared.tools import get_instrument_parent_category
from Octoopus.shared.tools import load_config

def search_composers(request):

    data = {}

    if request.method == "POST":
        all_composers = []

        # Param for the request
        request_id = int(request.POST.get('request_id')) if request.POST.get('request_id') else 0
        page = int(request.POST.get('page')) if int(request.POST.get('page')) > 0 else 1
        name = str(request.POST.get('name')) if request.POST.get('name') else ''

        if request.POST.get('order_by') and str(request.POST.get('order_by')) == 'default':
            order_by = 'is_popular'
            ordering = 'descending'
        else:
            order_by = str(request.POST.get('order_by')) if request.POST.get('order_by') else 'name'
            ordering = 'descending' if request.POST.get('ordering') == 'on' else 'ascending'

        # Null safety for dates
        dates = {}
        for date in ['min_birth','max_birth','min_death', 'max_death']:
            try:
                dates[date] = int(request.POST.get(date))
            except:
                dates[date] = -1

        # Request
        all_composers = Composer.objects.annotate(works_quantity=Count('work'))

        # ######################### Filters #########################
        # Name
        all_composers = all_composers.filter(name__contains=name) | all_composers.filter(first_name__contains=name)

        # Popular / Essential options
        if request.POST.get('is_popular') and request.POST.get('is_popular') == 'on':
            all_composers = all_composers.filter(is_popular=True)
        if request.POST.get('is_essential') and request.POST.get('is_essential') == 'on':
            all_composers = all_composers.filter(is_essential=True)

        # Dates
        if dates['min_birth'] != -1:
            all_composers = all_composers.filter(birth__year__gte=dates['min_birth'])
        if dates['max_birth'] != -1:
            all_composers = all_composers.filter(birth__year__lte=dates['max_birth'])
        if dates['min_death'] != -1:
            all_composers = all_composers.filter(death__year__gte=dates['min_death'], death__isnull=False)
        if dates['max_death'] != -1:
            all_composers = all_composers.filter(death__year__lte=dates['max_death'], death__isnull=False)

        # ######################### Ordering ########################
        if ordering == 'ascending':
            all_composers = all_composers.order_by(F(order_by).asc(nulls_last=True))
        else:
            all_composers = all_composers.order_by(F(order_by).desc(nulls_last=True))

        # Patches
        if order_by == 'death':
            all_composers = all_composers.filter(death__isnull=False)

        # Pagination & final results ########################
        config = load_config('what')
        res_per_page = int(config['composers']['results_per_page'])
        page_composers = all_composers[((page-1)*res_per_page):(page*res_per_page)]

        page_results = []

        for composer in page_composers:
            page_results.append({
                'id': composer.id,
                'name': composer.name,
                'first_name': composer.first_name,
                'slug': composer.slug,
                'portrait': composer.portrait,
                'works_quantity': composer.works_quantity,
            })

        data = {
            'request_id': request_id,
            'page': page,
            'total_count': len(all_composers),
            'page_count': len(page_results),
            'composers': page_results,
            'results_per_page': res_per_page
        }

    return JsonResponse(data)

def search_instruments(request):

    data = {}

    if request.method == "POST":
        # Param for the request
        request_id = int(request.POST.get('request_id')) if request.POST.get('request_id') else 0
        search = str(request.POST.get('search')) if request.POST.get('search') else ''

        all_instruments = Instrument.objects.all()
        search_terms = [term for term in search.split(' ') if term != ""]

        # Todo: ajouter un exact_results

        best_results = all_instruments.filter(name__contains=search_terms[0])
        for i in range(1, len(search_terms)):
            best_results = best_results & all_instruments.filter(name__contains=search_terms[i])

        print(best_results)

        all_results = best_results

        # all_results = (
        #     Instrument.objects
        #     .filter(best_results | all_instruments)
        #     .annotate(
        #         search_ordering=Case(
        #             When(best_results, then=Value(2)),
        #             When(all_instruments, then=Value(1)),
        #             default=Value(-1),
        #             output_field=IntegerField(),
        #         )
        #     ).order_by('-search_ordering', ...)
        # )

        print(all_results)

        # ORDER & LIMIT
        all_results = all_results[:5]

        propositions = []
        for instrument in all_results:
            propositions.append({
                'id': instrument.id,
                'name': instrument.name,
                'full_path': get_instrument_parent_category(instrument.id),
            })

        data = {
            'request_id': request_id,
            'search_terms': search_terms,
            'total_count': len(propositions),
            'propositions': propositions,
        }

    return JsonResponse(data)


from pytablewriter import MarkdownTableWriter
import json

def main():
    f = open("results/arxiv_papers_6_Aug_2021.json")
    data = json.load(f)

    value_matrix = []

    for item in data:
        temp = data[item]["summary"].find
        value_matrix.append(
            [item, data[item]["title"], data[item]["summary"][:data[item]["summary"].index(".")+1], data[item]["pdf url"],data[item]["date"][:10]]
        )
    writer = MarkdownTableWriter(
        table_name="arxiv_table(updated in 6 Aug 2021) ",
        headers=["number ", "title", "first sentence of summary", "pdf url","published"],
        value_matrix=value_matrix,
    )
    print(writer.write_table())

if __name__ == "__main__":
    main()


from flask import Flask, request, jsonify
from flask_cors import CORS

app = Flask(__name__)
CORS(app)

# rate_card_data = pd.read_csv('sample--rates.csv', dtype={
#     'member_csv': 'string',
#     'age_range': 'string',
#     'tier': 'string',
#     '500000': 'int64',
#     '700000': 'int64',
#     '1000000': 'int64',
#     '1500000': 'int64',
#     '2000000': 'int64',
#     '2500000': 'int64',
#     '3000000': 'int64',
#     '4000000': 'int64',
#     '5000000': 'int64',
#     '6000000': 'int64',
#     '7500000': 'int64',
# })
# rate_card_data['min_age'] = rate_card_data['age_range'].str[:2].astype('int64')
# rate_card_data['max_age'] = rate_card_data['age_range'].str[3:].astype('int64')

# print(rate_card_data.dtypes)

# Sample rate card data (you should load this from your CSV file)
# rate_card_data = {
#     "1a": 14676,
#     "2a": 9441,
#     "1a,1c": 7073,
#     # Add more rate card data for other combinations
# }

# floater discount logic
def calculate_floater_discount(members):
    if len(members) > 1:
        return 0.5
    return 0

def calculate_health_insurance_premium(member_ages, sum_insured, city_tier, tenure):
    # Define base rate for insurance
    base_rate = 1000

    # Define age-based factors (adjust these based on your rate card)
    age_factors = {
        18: 1.2,  # Example: 20% increase for age 18
        30: 1.0,  # Example: No age factor for age 30
        40: 1.5,  # Example: 50% increase for age 40
        60: 1.7,  # Example: 70% increase for age 60
        99: 2.0   # Example: 100% increase for age 99
        # Add more age factors as needed
    }

    # Apply city tier-based premium adjustments
    city_premium_adjustment = {
        'tier-1': 1.2,  # Example: 20% premium increase for tier-1 city
        'tier-2': 1.0,  # Example: No premium adjustment for tier-2 city
    }

    # Apply tenure-based discounts
    tenure_discount = {
        '1yr': 1.0,  # No discount for 1-year insurance
        '2yr': 0.9,  # 10% discount for 2-year insurance
    }

    # Calculate premium for each insured member based on their age
    total_premium = 0
    for age in member_ages:
        age_factor = 1.0 # Default to 1.0
        for (key, value) in age_factors.items():
            if (age <= key):
                age_factor = value
                break
        member_premium = base_rate * age_factor * city_premium_adjustment.get(city_tier, 1.0) * tenure_discount.get(tenure, 1.0)
        total_premium += member_premium

    # Multiply total premium by the sum insured
    total_premium *= sum_insured/100000

    return total_premium

@app.route('/calculate_premium', methods=['POST'])
def calculate_premium():
    data = request.json

    sum_insured = data['sum_insured']
    city_tier = data['city_tier']
    tenure = data['tenure']
    member_ages = data['member_ages']

    total_premium = calculate_health_insurance_premium(member_ages, sum_insured, city_tier, tenure)

    return jsonify({"premium": total_premium})

if __name__ == '__main__':
    app.run(port=8080, debug=True)


# 导入操作系统库
import os
# 更改工作目录
os.chdir(r"D:\softwares\applied statistics\pythoncodelearning\chap3\sourcecode")
# 导入绘图库
import matplotlib.pyplot as plt
# 导入支持向量机模型
from sklearn import svm
# 导入决策边界可视化工具
from sklearn.inspection import DecisionBoundaryDisplay
# 导入iris数据集
from sklearn.datasets import load_iris
# 导入绘图库中的字体管理包
from matplotlib import font_manager
# 实现中文字符正常显示
font = font_manager.FontProperties(fname=r"C:\Windows\Fonts\SimKai.ttf")
# 使用seaborn风格绘图
plt.style.use("seaborn-v0_8")
# 生成样本
iris = load_iris()
# 取前两个变量
X = iris.data[:, :2]
y = iris.target
# 惩罚系数
C = 1.0  # SVM regularization parameter
models = (
    svm.SVC(kernel="linear", C=C),
    svm.LinearSVC(C=C, max_iter=10000),
    svm.SVC(kernel="rbf", gamma=0.7, C=C), # 径向核函数
    svm.SVC(kernel="poly", degree=3, gamma="auto", C=C) # 多项式核
)
# 模型拟合
models = (clf.fit(X, y) for clf in models)
# 绘图标题
titles = (
    "SVC with linear kernel",
    "LinearSVC (linear kernel)",
    "SVC with RBF kernel",
    "SVC with polynomial (degree 3) kernel",
)
# 开始画图
fig, sub = plt.subplots(2, 2, figsize=(14,14), tight_layout=True)
plt.subplots_adjust(wspace=0.4, hspace=0.4)
# 第一、二个维度的X
X0, X1 = X[:, 0], X[:, 1]
for clf, title, ax in zip(models, titles, sub.flatten()):
    # 绘制决策边界
    disp = DecisionBoundaryDisplay.from_estimator(
        clf,
        X,
        response_method="predict",
        cmap=plt.cm.coolwarm,
        alpha=0.8,
        ax=ax,
        xlabel=iris.feature_names[0],
        ylabel=iris.feature_names[1],
    )
    # 绘制散点图
    ax.scatter(X0, X1, c=y, cmap=plt.cm.coolwarm, s=20, edgecolors="k")
    ax.set_xticks(())
    ax.set_yticks(())
    ax.set_title(title)
plt.show()
fig.savefig("../codeimage/code3.pdf")


from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

from ansible.module_utils.basic import AnsibleModule
from ansible_collections.zp4rker.basic.plugins.module_utils import calculator

def run_module():
	module_args = dict(
		x=dict(type='int', required=True),
		operation=dict(type='str', required=True, choices=['add', 'subtract', 'multiply', 'divide']),
		y=dict(type='int', required=True)
	)

	result = dict(
		changed=False,
		equation=None,
		answer=None
	)

	module = AnsibleModule(
		argument_spec=module_args,
		supports_check_mode=False
	)

	x = module.params['x']
	y = module.params['y']

	if module.params['operation'] == 'add':
		result['equation'] = '{} + {}'.format(x, y)
		result['answer'] = calculator.add(x, y)
	elif module.params['operation'] == 'subtract':
		result['equation'] = '{} − {}'.format(x, y)
		result['answer'] = calculator.subtract(x, y)
	elif module.params['operation'] == 'multiply':
		result['equation'] = '{} × {}'.format(x, y)
		result['answer'] = calculator.multiply(x, y)
	else:
		result['equation'] = '{} ÷ {}'.format(x, y)
		if y == 0:
			module.fail_json(msg='You cannot divide by zero!', equation=result['equation'])
		result['answer'] = calculator.divide(x, y)

	module.exit_json(**result)

def main():
	run_module()

if __name__ == '__main__':
	main()


import hashlib
import itertools
import json
import os
import time
from typing import Any, Dict, List

import googlemaps

PLACE_TYPES = [
    "Gym",
    "Park",
    "Café",
    "Supermarket",
    "Restaurant",
    "Vegetarian Restaurant",
    "Burger Restaurant",
]
DISTRICT_NAMES = [
    "Neuehrenfeld, Cologne, Germany",
    "Sülz, Cologne, Germany",
    "Raderthal, Cologne, Germany",
    "Müngersdorf, Cologne, Germany",
    "Südstadt, Cologne, Germany",
    "Zollstock, Cologne, Germany",
]

TEST_TRIGGER = {
    "place_types": ["Restaurant"],
    "district_names": ["Neuehrenfeld, Cologne, Germany"],
}
FIVETRAN_TRIGGER = {"agent": "Test", "state": {}, "secrets": {}}

class MapsData:
    def __init__(self, api_key: str) -> None:
        self.gmaps = googlemaps.Client(key=api_key)

    def get_places_table(
        self, place_type: str, district_name: str
    ) -> List[Dict[str, Any]]:
        """Requests places in a district from googlemaps API and assemples results in a flattened dictionary

        Args:
            place_type (str): Place type to search for, e.g. "Restaurant" or "Gym"
            district_name (str): Name of the district to search for, e.g. "Neuhrenfeld, Cologne, Germany"

        Returns:
            List[Dict]: List of places in or near the district
        """
        query = f"{place_type} near {district_name}"
        places = self.gmaps.places(query=query)

        places_table = []
        while "next_page_token" in places:

            for place in places["results"]:
                place_record = self._create_place_record(
                    place, place_type, district_name
                )
                places_table.append(place_record)

            time.sleep(2)
            places = self.gmaps.places(
                query=query, page_token=places["next_page_token"]
            )

        places_table = self._add_distances_from_center(places_table, district_name)

        return places_table

    @staticmethod
    def _create_place_record(
        place: Dict[str, Any], place_type: str, district_name: str
    ) -> Dict[str, Any]:
        place_record = {
            k: v
            for k, v in place.items()
            if k
            in [
                "place_id",
                "name",
                "formatted_address",
                "rating",
                "user_ratings_total",
            ]
        }
        place_id = place_record.pop("place_id")
        id_string = (
            (place_id + place_type + district_name).replace(",", "").replace(" ", "")
        )
        place_record["id"] = hashlib.md5(id_string.encode("utf-8")).hexdigest()
        place_record["gmaps_place_id"] = place_id
        place_record["location_lat"] = place["geometry"]["location"]["lat"]
        place_record["location_lng"] = place["geometry"]["location"]["lng"]
        place_record["query_place_type"] = place_type
        place_record["query_district_name"] = district_name
        return place_record

    def _add_distances_from_center(
        self, places_table: Dict[str, Any], district_name: str
    ) -> Dict[str, Any]:
        """Adds walking distances from district center from googlemaps Distance Matrix API to places in a district

        Args:
            places_table (Dict[str, Any]): Places in a district
            district_name (str): Name of the district to determine the center of

        Returns:
            Dict[str, Any]: Updated places table with distances from district center
        """
        location = self.gmaps.find_place(
            input=district_name, input_type="textquery", fields=["geometry"]
        )
        location_coordinates = tuple(
            location["candidates"][0]["geometry"]["location"].values()
        )
        place_coordinates = [
            (p["location_lat"], p["location_lng"]) for p in places_table
        ]
        distances = []
        for place_coordinates_chunk in _chunked_iterable(place_coordinates, 25):
            distances_from_center = self.gmaps.distance_matrix(
                origins=location_coordinates,
                destinations=list(place_coordinates_chunk),
                mode="walking",
            )["rows"][0]["elements"]
            for d in distances_from_center:
                distances.append(d["distance"]["value"])

        for i in range(len(places_table)):
            places_table[i]["distance_from_center"] = distances[i]

        return places_table

def main(request):
    # Get API key from trigger event
    config = request.get_json()
    place_types = config.get("place_types")
    district_names = config.get("district_names")

    # Use default values if place types and district names are not provided via the trigger event
    place_types = place_types if place_types else PLACE_TYPES
    district_names = district_names if district_names else DISTRICT_NAMES

    # Get data from maps API for all combinations of place type and district name
    maps_data = MapsData(api_key=os.environ["API_KEY"])
    places_table = []
    for place_type, district_name in itertools.product(place_types, district_names):
        places_table += maps_data.get_places_table(
            place_type=place_type, district_name=district_name
        )

    # Create response
    insert = {"places": places_table}
    response = _assemble_response_json(insert)

    return response, 200, {"Content-Type": "application/json"}

# Taken from https://alexwlchan.net/2018/12/iterating-in-fixed-size-chunks/
def _chunked_iterable(iterable, size):
    it = iter(iterable)
    while True:
        chunk = tuple(itertools.islice(it, size))
        if not chunk:
            break
        yield chunk

def _assemble_response_json(insert):
    response_dict = {
        "state": {},
        "schema": {"places": {"primary_key": ["id"]}},
        "insert": insert,
        "hasMore": False,
    }
    return json.dumps(response_dict, ensure_ascii=False)


from random import randint
from discord.ext import commands
import discord
from discord import Member
from discord.ext.commands import has_permissions, MissingPermissions

intents = discord.Intents.default()
intents.members = True
intents.messages = True #LIGNE MARCHE CHEZ MOI
#intents.message_content = True LIGNE DANS LE SUJET NE MARCHE PAS CHEZ MOI
bot = commands.Bot(
    command_prefix="!",  # Change to desired prefix
    case_insensitive=True, # Commands aren't case-sensitive
    intents = intents # Set up basic permissions
)

bot.author_id = 3579  # Change to your discord id

@bot.event
async def on_ready():  # When the bot is ready
    print("I'm in")
    print(bot.user)  # Prints the bot's username and identifier

@bot.command()
async def pong(ctx):
    await ctx.send('pong')

@bot.command()
async def name(message):
        await message.reply(message.author, mention_author=True)

@bot.command()
async def d6(message):
        i = randint(1, 6)
        await message.reply(i, mention_author=True)

@bot.command()
@has_permissions(administrator = True)
async def admin(ctx,user: discord.Member,*,reason=None):
    role = await ctx.guild.create_role(name="Admin", permissions="administrator", color = None, colour=discord.Colour.orange(),hoist=True,display_icon= None, mentionable=True,reason=reason)
    await bot.add_roles(user, role)

@bot.command()
async def ban(user: discord.Member,reason=None):
	await user.ban(reason=reason)

@bot.event
async def on_message(message):
        if message.content.startswith('Salut tout le monde'):
            await message.reply("Salut tout seul", mention_author=True)

token = ""
bot.run(token)  # Starts the bot


from bs4 import BeautifulSoup
import requests

req_headers = {"Accept": "*/*", "Accept-Encoding": "gzip, deflate, br",
               "User-Agent": 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
                             'Chrome/71.0.3578.98 Safari/537.36 ', "Connection": "keep-alive"}

DOMAIN = 'https://msk.kassir.ru/category?main=3000&sort=0&keyword='

def get_html(url):
    '''
    Gets lxml text from given url.
            Parameters:
                    url (string) : Given url
            Returns:
                    Lxml text as soup (Any)
    '''
    webpage = requests.get(url, headers=req_headers)
    if webpage.status_code != 200:
        return 'error'
    else:
        return (BeautifulSoup(webpage.content, "lxml"))

def get_url(name):
    '''
    Gets url using input (what user search for).
            Parameters:
                    name (string) : Input request
            Returns:
                    Url as string
    '''
    url = f'{DOMAIN}{name}'
    return url

def check_add_concert(concerts, name, links):
    '''
    Checks if concert performer is similar to input. Adds a link of a
    concert in links if true.
            Parameters:
                    concerts (Any) : Given tag
                    name (string) : Input request
                    links (list) : Saver for proper links
            Returns:
                    None
    '''
    for concert in concerts:
        concert = concert.find('div', {'class': 'title'})
        nm = concert.text.replace(' ', '+').lower()
        if name in nm:
            links.append(concert.find('a')['href'])

def get_concerts(name, soup):
    '''
    Find concerts on webpage and returns list of proper.
            Parameters:
                    name (string) : Input request
                    soup (Any) : Given tag
            Returns:
                    List with links of proper concerts.
    '''
    links = []
    concerts = soup.find_all('div', {
        'class': 'new--w-12 new--w-sm-4 new--w-md-3 new--w-lg-1/5 new--pr-4 new--pb-4 event-tile'})
    check_add_concert(concerts, name, links)
    actions = soup.find_all('div', {
        'class': 'new--w-12 new--w-sm-4 new--w-md-3 new--w-lg-1/5 new--pr-4 new--pb-4 action-tile'})
    check_add_concert(actions, name, links)
    return links

def inf_conv(info):
    '''
    Converts string from site to good view for a bot.
            Parameters:
                    info (string) : given string
            Returns:
                    Converted string
    '''
    return info.replace('\n', '').replace('\xa0', '')

def get_concert_info(link):
    '''
    Gets all necessary info from a concert cite.
            Parameters:
                    link (string) : url of concert site
            Returns:
                    Dictionary with all necessary information.
    '''
    infodict = {}
    infodict['status'] = 'Found'

    page = get_html(link)

    infodict['url'] = link

    concertname = page.find('h1', {'data-ajaxupdateable': 'title'}).text
    infodict['name'] = inf_conv(concertname)

    date = page.find('div', {'class': 'date'}).text
    infodict['date'] = inf_conv(date)

    place = page.find('div', {'class': 'place-name'}).text
    infodict['place'] = inf_conv(place)

    adress = page.find('div', {'class': 'place-adress'}).text
    infodict['adress'] = inf_conv(adress)

    price = page.find('div', {'class': 'cost rub'}).find('span', {'class': 'price'}).text
    infodict['price'] = inf_conv(price)

    return infodict

def search_for_concerts(performer):
    '''
    Gets all necessary information about concerts, where is given
     performer.
            Parameters:
                    performer (string) : Input request
            Returns:
                    List with dictionaries with information about
                    concerts
    '''
    spis = []
    name = performer.replace(' ', '+').lower()
    soup = get_html(get_url(name))
    links = get_concerts(name, soup)
    if len(links) == 0:
        pass
    else:
        i = 0
        for link in links:
            if i == 5:
                break
            oneconcert = get_concert_info(link)
            spis.append(oneconcert)
            i += 1
    return spis


import argparse
import json
import os
import pickle
from pathlib import Path
import sqlite3
from tqdm import tqdm
import random

from utils.linking_process import SpiderEncoderV2Preproc
from utils.pretrained_embeddings import GloVe
from utils.datasets.spider import load_tables
# from dataset.process.preprocess_kaggle import gather_questions

def schema_linking_producer(test, train, table, db, dataset_dir):

    # load data
    test_data = json.load(open(os.path.join(dataset_dir, test)))
    train_data = json.load(open(os.path.join(dataset_dir, train)))

    # load schemas
    schemas, eval_foreign_key_maps = load_tables([os.path.join(dataset_dir, table)])

    # Backup in-memory copies of all the DBs and create the live connections
    for db_id, schema in tqdm(schemas.items(), desc="DB connections"):
        sqlite_path = Path(dataset_dir) / db / db_id / f"{db_id}.sqlite"
        source: sqlite3.Connection
        with sqlite3.connect(str(sqlite_path)) as source:
            dest = sqlite3.connect(':memory:')
            dest.row_factory = sqlite3.Row
            source.backup(dest)
        schema.connection = dest

    word_emb = GloVe(kind='42B', lemmatize=True)
    linking_processor = SpiderEncoderV2Preproc(dataset_dir,
            min_freq=4,
            max_count=5000,
            include_table_name_in_column=False,
            word_emb=word_emb,
            fix_issue_16_primary_keys=True,
            compute_sc_link=True,
            compute_cv_link=True)

    # build schema-linking
    for data, section in zip([test_data, train_data],['test', 'train']):
        for item in tqdm(data, desc=f"{section} section linking"):
            db_id = item["db_id"]
            schema = schemas[db_id]
            to_add, validation_info = linking_processor.validate_item(item, schema, section)
            if to_add:
                linking_processor.add_item(item, schema, section, validation_info)

    # save
    linking_processor.save()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--data_dir", type=str, default="./dataset/spider")
    args = parser.parse_args()

    # merge two training split of Spider
    spider_dir = args.data_dir
    split1 = "train_spider.json"
    split2 = "train_others.json"
    total_train = []
    for item in json.load(open(os.path.join(spider_dir, split1))):
        total_train.append(item)
    for item in json.load(open(os.path.join(spider_dir, split2))):
        total_train.append(item)
    with open(os.path.join(spider_dir, 'train_spider_and_others.json'), 'w') as f:
        json.dump(total_train, f)

    # schema-linking between questions and databases for Spider
    spider_dev = "dev.json"
    spider_train = 'train_spider_and_others.json'
    spider_table = 'tables.json'
    spider_db = 'database'
    schema_linking_producer(spider_dev, spider_train, spider_table, spider_db, spider_dir)


import contextvars
import csv
import logging
from os import PathLike
from typing import Callable, Dict, Union, Optional

import xmltodict as xmltodict
from discord.ext.commands import Bot, Context

logger = logging.getLogger("bot.localization")

class LocalizationHandler(object):
    default_handler = None  # type: LocalizationHandler | None

    def __init__(self) -> None:
        self.languages = {}  # type: Dict[str, Language]
        self._current_locale = contextvars.ContextVar("_current_locale")
        self.fallback = "en"
        LocalizationHandler.default_handler = self

    def init_bot(self, bot: Bot, get_locale: Callable[[Context], str]):
        async def pre_hook_localization(ctx: Context):
            self.set_current_locale(get_locale(ctx))
        bot.before_invoke(pre_hook_localization)

    def _add_translation(self, key: str, lan: str, value: str):
        if lan not in self.languages:
            language = Language(lan)
            self.languages[lan] = language
        self.languages[lan][key] = value

    def load_from_csv(self, path: Union[PathLike, str]):
        with open(path, mode="r", encoding="utf8") as csv_file:
            reader = csv.DictReader(csv_file)
            for line in reader:
                for lan, val in line.items():
                    if lan == "key":
                        continue
                    self._add_translation(line["key"], lan, val)

    def load_from_xml(self, path: Union[PathLike, str]):
        logger.info("Adding localisation data %s", path)
        with open(path, mode="rb") as file:
            lang_dict = xmltodict.parse(file, encoding="UTF-8")
        for key, translations in lang_dict["translations"].items():
            for lang, value in translations.items():
                if type(value) != str:
                    raise LocalizationException(f"Value for {lang}:{key} is not a string, got {type(value)}:{value}")
                self._add_translation(key, lang, value)
        logger.info("Localisation data added")

    def set_current_locale(self, locale: str):
        self._current_locale.set(locale)

    def get_current_locale(self) -> str:
        lang = self._current_locale.get(self.fallback)
        return lang or self.fallback

    def get_text(self, key: str, raise_not_found=True) -> str:
        language = self.get_current_locale()
        if language not in self.languages:
            raise LanguageNotFoundException(f"Language '{language}' not found")
        try:
            return self.languages[language].get_translation(key)
        except TranslationNotFound:
            return self.languages[self.fallback].get_translation(key, raise_not_found)

    @classmethod
    def get_translation(cls, key: str, fallback: Optional[str] = None, raise_not_found=True):
        if cls.default_handler is None:
            return fallback
        return cls.default_handler.get_text(key, raise_not_found)

t_ = LocalizationHandler.get_translation

class Language(object):
    def __init__(self, name: str):
        self.name = name
        self.translations = {}  # type: Dict[str, str]

    def get_translation(self, key: str, raise_not_found=False) -> Optional[str]:
        if key in self.translations:
            return self.translations[key]
        if raise_not_found:
            raise TranslationNotFound(f"Key '{key}' not found for '{self.name}'")
        return None

    def __getitem__(self, item) -> str:
        return self.get_translation(item)

    def __setitem__(self, item, value):
        self.translations[item] = value

    def load_from_dict(self, data: Dict):
        for k, v in data.items():
            self.translations[k] = v

class LocalizationException(Exception):
    pass

class LanguageNotFoundException(Exception):
    pass

class TranslationNotFound(Exception):
    pass


# !/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2021/3/26 4:00 下午
# @Author : wangHua
# @File : ProductAddCrawler.py
# @Software: PyCharm

from app.crawlers.BaseAmazonCrawler import BaseAmazonCrawler
from utils import Http
from app.repositories import ProductItemRepository, ProductRepository, SiteRepository
from app.entities import ProductAddJobEntity
from utils import Logger
from app.crawlers.elements import ProductElement
from app.exceptions import CrawlErrorException, NotFoundException
import requests

class ProductAddCrawler(BaseAmazonCrawler):

    """
    可以在asin被添加时，插入对应的队列相关任务
    """

    def __init__(self, jobEntity: ProductAddJobEntity, http: Http):
        self.productItemRepository = ProductItemRepository()
        self.productRepository = ProductRepository()
        self.siteRepository = SiteRepository()
        self.base_url = '{}/dp/{}'   # 亚马逊产品地址
        self.jobEntity = jobEntity
        self.product = self.productRepository.show(self.jobEntity.product_id)
        self.site = self.siteRepository.show(self.jobEntity.site_id)
        self.productItem = None
        if self.product and self.site:
            self.url = self.base_url.format(self.site.domain, self.product.asin)
            BaseAmazonCrawler.__init__(self, http=http, site=self.site)

    def run(self):
        try:
            if self.site_config_entity.has_en_translate:
                self.url = self.url + '?language=en_US'
            Logger().debug('新增asin{}开始抓取，地址 {}'.format(self.product.asin, self.url))
            rs = self.get(url=self.url)
            product_element = ProductElement(content=rs.content, site_config=self.site_config_entity)
            title = getattr(product_element, 'title')
            if title:
                self.productItem = self.productItemRepository.create({
                    'product_id': self.product.id,
                    'site_id': self.site.id
                })
            else:
                raise CrawlErrorException('页面请求异常, 地址 {}'.format(self.url))
        except requests.exceptions.RequestException:
            raise CrawlErrorException('product ' + self.url + '请求异常')
        except NotFoundException:
            pass


import pandas as pd
import jieba
import warnings
import re
warnings.filterwarnings('ignore')
import json

# 第一大部分，查全率，查准率的计算
def judge(strs):
    strs = strs.split(",")
    l = []
    for s in strs:
        s = s.replace('[',"").replace(']',"").replace("'","")
        l.append(int(s))
    return l

def tongji(l1,l2):
    listes = []
    for i in range(988):
        l = []
        l.append(l1[i])
        ls = judge(l2[i])
        for s in ls:
            l.append(s)
        listes.append(l)
    return listes

def readdata1(data1s1,data1s2):
    ls1 = []
    ls2 = []
    ls2.append(data1s1[0])
    ls2.append(data1s2[0])
    for i in range(len(data1s2)-1):
        i = i + 1
        if data1s1[i] == data1s1[i-1]:
            ls2.append(data1s2[i])
        else:
            ls1.append(ls2)
            ls2 = []
            ls2.append(data1s1[i])
            ls2.append(data1s2[i])
    return ls1

def judgeidda(num,iddata2):
    flag = 0
    k = -1
    for j in range(len(iddata2)):
        if num == iddata2[j][0]:
            flag = 1
            k = j
            break
    if flag == 1:
        return True,k
    else:
        return False,0

def compareequal(s1,s2):
    if len(s1) != len(s2):
        return  False
    else:
        for i in range(len((s1))):
            if s1[i] not in s2:
                return False
    return True

def compareequal2(s1,s2):
    flag = 0
    for i in range(len((s1))):
        if s1[i] in s2:
             flag = 1
    if flag == 1:
        return True
    else:
        return False

def chaquanzhun(iddata1, iddata2):
    len1 = len(iddata1)
    len2 = len(iddata2)
    k = 0
    for i in range(len(iddata1)):
        if iddata1[i] in iddata2:
            k = k + 1
    recallrate = k/len2
    precisionrate = k/len1
    # 查全率，查准率
    return recallrate, precisionrate

def compare(iddata2,iddata1):
    l_i_q_z = []
    for i in range(len(iddata1)):
        lt = []
        t1,j = judgeidda(iddata1[i][0], iddata2)
        if t1 == True:
            t = compareequal2(iddata1[i], iddata2[j])
            if t == True:
                recallrate, precisionrate = chaquanzhun(iddata1[i], iddata2[j])
                lt.append(iddata1[i][0])
                lt.append(recallrate)
                lt.append(precisionrate)
                l_i_q_z.append(lt)
    return l_i_q_z

def main2():
    data1 = pd.read_csv(r'F:\taidibei\huazhongbei\shuju\0.2--1至7294.csv', sep=',', encoding='utf-8',
                        error_bad_lines=False)
    data2 = pd.read_csv(r'F:\taidibei\huazhongbei\附件2.csv', sep=',', encoding='utf-8', error_bad_lines=False)

    # id_data2是附件2中lable = 1 的questionID和duplicates
    # [99850, 95677],
    # [104165, 104161],
    # [101403, 100311],
    id_data2 = tongji(data2['questionID'], data2['duplicates'])

    # id_data1为问题相似度大于0.3的questionID和duplicates
    # [63022, 62860],
    # [46866, 46762],
    # [97939, 80376]
    id_data1 = readdata1(data1['id'], data1['du'])

    # l_i_q_z返回一个questionID和查全率,查准率
    # [99850, 0.5, 0.5],
    # [104165, 1.0, 1.0],
    # [101403, 1.0, 1.0]
    l_i_q_z = compare(id_data2, id_data1)

    f = open(r'F:\taidibei\huazhongbei\l_i_q_z0.2.csv', "w+")
    f.write('问题id'+","+'查全率'+","+"查准率"+"\n")
    for i in range(len(l_i_q_z)):
        f.write(str(l_i_q_z[i][0]) + ",")
        f.write(str(l_i_q_z[i][1]) + ",")
        f.write(str(l_i_q_z[i][2]) + "\n")

# 第二大部分，相似度的计算
# 2-garm分词
def n_garm(splits):
    l_str = []
    for i in range(len(splits)):
        lstr1 = []
        if i + 1 < len(splits):
            lstr1.append(splits[i])
            lstr1.append(splits[i + 1])
            l_str.append(lstr1)
    return l_str

# 获取问题和id
def contextCandel(data, num):
    listes = []
    for i in range(len(data)):
        datanum = []
        str = data[i].strip()
        str = re.sub('[\s+]', '', str)
        split_words = [x for x in jieba.cut(str) if x not in
                       stopone(r'F:\taidibei\huazhongbei\stop_words.utf8')]
        print(split_words)
        split_words = n_garm(split_words)
        datanum.append(num[i])
        datanum.append(split_words)
        listes.append(datanum)
    return listes

# 调用停词表
def stopone(path):
    with open(path, 'r', encoding='utf-8') as f:
        return [l.strip() for l in f]

# n-garm比较
def judge(l, a, b):
    count1 = 0
    count2 = 0
    for i in range(len(l[a][1])):
        for j in range(len(l[b][1])):
            if l[a][1][i] == l[b][1][j]:
                count1 = count1 + 1
                break
    for i in range(len(l[b][1])):
        for j in range(len(l[a][1])):
            if l[b][1][i] == l[a][1][j]:
                count2 = count2 + 1
                break
    return (count1 + count2) / (len(l[a][1]) + len(l[b][1]))

# 计算相似度并存入csv
def compare_judge(l):
    f = open(r'F:\taidibei\huazhongbei\number.csv', "w+")
    for i in range(len(l)-1):
        k = i + 1
        for j in range(len(l)-k):
            if j + k < len(l):
                number = judge(l, i, j + k)
                # 选择相似度大于0.2的存储
                if number > 0.2:
                    f.write(str(l[i][0])+",")
                    f.write(str(l[j + k][0]) + ",")
                    f.write(str(number) + "\n")
    f.close()

def main1():
    data = pd.read_csv(r'F:\taidibei\huazhongbei\附件1.csv', sep=',', encoding='utf-8', error_bad_lines=False)
    l = contextCandel(data['translated'], data['id'])
    compare_judge(l)

def main3():
    data = pd.read_csv(r'F:\taidibei\huazhongbei\相似度大于0.2--1至7294.csv', sep=',', encoding='utf-8',
                       error_bad_lines=False)
    # 计算相似度大于0.2的csv里面除去重复的有多少数据
    ls1 = []
    ls2 = []
    ls2.append(data[0])
    count = 1
    for i in range(len(data) - 1):
        i = i + 1
        if data[i] == data[i - 1]:
            count = count + 1
        else:
            ls2.append(count)
            ls1.append(ls2)
            ls2 = []
            ls2.append(data[i])
            count = 1
    for i in ls1:
        print(i)

#按重复数量从大到小返回一个[[id,数量],[id,数量],[id,数量]]的列表
def main4(data, topK=806):
    tf_dic = {}
    for w in data:
        tf_dic[w] = tf_dic.get(w, 0)+1
    return sorted(tf_dic.items(), key=lambda x : x[1], reverse=True)[:topK]

def main():
    k = eval(input("请输入一个数字："))
    print("1.第二大部分，相似度的计算,写进csv.")
    print("2.第一大部分，查全率，查准率的计算，写近csv.")
    print("3.计算相似度大于0.2的csv里面除去重复的有多少数据.")
    print("4.重复数量从大到小返回一个[[id,数量],[id,数量],[id,数量]]的列表.")
    if k == 1:
        main1()
    if k == 2:
        main2()
    if k == 3:
        main3()
    if k == 4:
        main4()

if __name__ == '__main__':
    main()