from time import time

import glob
from sys import argv
import cv2
import pickle

import shutil
from sklearn.cluster import KMeans
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score, confusion_matrix
from sklearn import preprocessing, svm
import numpy as np

## Inputs

k1 = 50
relearnK = False

c1 = 20
svc_kernel = 'rbf' # 'linear'

working_dir = ".\\images"

# phase subdirs
learn_subdir       = "train"
test_subdir        = "test"
validation_subdir  = "valid"

# classes
cat1 = "chat4"
cat2 = "chien3"
cat3 = "formule1"
cat4 = "motoGrandPrix"

def getCatPath(cat_name,extension,phase_subdir):
    return ("%s\\%s\\%s"+"/*.%s") % (working_dir,phase_subdir,cat_name,extension)

baryName = ".\\data\\" # saved files dir

# verbose options

verbose_params = True

verbose_steps = True
verbose_files = False
verbose_state = False

verbose_kmeans = False
verbose_score = True
verbose_confusion = True

## Sequence variables

LEARN = 0
VALIDATE = 1
TEST = 2

stepNameDict = {LEARN: "train",   VALIDATE: "valid",      TEST: "test"}
subDirDict   = {LEARN: learn_subdir, VALIDATE: validation_subdir, TEST: test_subdir}

SEQUENCE = [LEARN, VALIDATE, TEST]

## Running sequence

if (verbose_params):
    print("k1 = %d"%(k1))
    print("c1 = %3f"%(c1))
    print("kernel = %s"%(svc_kernel))

debug = False

for STEP in SEQUENCE:

    if (verbose_steps):
        print("\n"+stepNameDict[STEP]+"...\n")

    subdir = subDirDict[STEP]

    ## Establishing groundTruth
    path_cat_1 = getCatPath(cat1,"jpg",subdir)
    path_cat_2 = getCatPath(cat2,"jpg",subdir)
    path_cat_3 = getCatPath(cat3,"jpeg",subdir)
    path_cat_4 = getCatPath(cat4,"jpeg",subdir)

    if (verbose_files):
        print("Reading" + 4*"\n\t%s" % (path_cat_1,path_cat_2,path_cat_3,path_cat_4))

    listImg = glob.glob(path_cat_1)
    tmp1 = len(listImg)
    groundTruth = [0]*tmp1

    listImg += glob.glob(path_cat_2)
    tmp2 = len(listImg) - tmp1
    groundTruth += [1]*tmp2

    listImg += glob.glob(path_cat_3)
    tmp3 = len(listImg) - ( tmp1 + tmp2 )
    groundTruth += [2]*tmp3

    listImg += glob.glob(path_cat_4)
    tmp4 = len(listImg) - ( tmp1 + tmp2 + tmp3 )
    groundTruth += [3]*tmp4

    if (verbose_files): print("Done reading" + 4*" %d" % (tmp1, tmp2, tmp3, tmp4))

    ## retrieving SIFT descriptors

    dimImg = [] # nb of sift per file
    lesSift = np.empty(shape=(0, 128), dtype=float) # array of all SIFTS from all images

    for s in listImg:
        if (verbose_files):
            print("###",s,"###")
        image = cv2.imread(s)
        gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
        sift = cv2.xfeatures2d.SIFT_create()
        kp,des = sift.detectAndCompute(gray,None)
        if (verbose_files):
            print("SIFT: ", len(kp))
        dimImg.append(len(des))
        lesSift = np.append(lesSift,des,axis=0)

    ## Computing k-means

    if (verbose_state): print("Done sifting")

    sifts = lesSift ###### preprocessing.normalize(lesSift)

    if ((STEP==LEARN) and relearnK):
        t0 = time()
        kmeans1 = KMeans(n_clusters=k1, random_state=0).fit(sifts)
        print("KMeans completed in %0.3fs" % (time() - t0))

        labels = kmeans1.labels_

        #sauvegarde de l'objet
        with open(baryName+'multiclass.bary', 'wb') as output:
            pickle.dump(kmeans1, output, pickle.HIGHEST_PROTOCOL)

    if (STEP==VALIDATE or STEP==TEST or (not relearnK)):
        #chargement de l'objet
        with open(baryName+'multiclass.bary', 'rb') as input:
            kmeans1 = pickle.load(input)

        labels = kmeans1.predict(sifts)

    if (verbose_kmeans): print("result of kmeans k=%d"%(k1), labels)

    ## Writing BOWs from k-means result

    bows = np.empty(shape=(0,k1),dtype=float) #BOW initialization

    i = 0
    for nb in dimImg: # for each sound (file)
        tmpBow = [0]*k1
        j = 0
        while j < nb: # for each SIFT of this sound (file)
            tmpBow[labels[i]] += 1
            j+=1
            i+=1
        copyBow = tmpBow.copy()
        bows = np.append(bows, [copyBow], 0)

    # ** Here was the culprit **
    BOWs = bows ###### preprocessing.normalize(bows)

    ## Classification
    # Todo : Tweak 'decision_function_shape'
    classif = svm.SVC(C=c1,kernel=svc_kernel, decision_function_shape='ovr')

    if (STEP==LEARN):
        classif.fit(BOWs, groundTruth) # apprentissage
        #sauvegarde de l'objet
        with open(baryName+'multiclassSVC.svm', 'wb') as output:
            pickle.dump(classif, output, pickle.HIGHEST_PROTOCOL)

    if (STEP==VALIDATE or STEP==TEST):
        #chargement de l'objet
        with open(baryName+'multiclassSVC.svm', 'rb') as input:
            classif = pickle.load(input)

    if (verbose_score):
        print("groundTruth: ", np.array(groundTruth))
        res = classif.predict(BOWs)
        print("prediction : ",res)

        # Todo : Tweak 'average'
        F1 = f1_score(groundTruth, res, average=None) #F1
        print("F-measure =",F1)

        score = classif.score(BOWs, groundTruth)
        print(stepNameDict[STEP],"score = ", score)

    if (verbose_confusion):
        confusion = confusion_matrix(groundTruth, res, labels=[0,1,2,3])
        print(confusion)


# geocoding

from zope import interface, component

from getpaid.core.interfaces import IStoreSettings, IShippableLineItem, IOrder, IOriginRouter
from getpaid.core.payment import ContactInformation

import interfaces

class OriginRouter( object ):

    " warehouse aware origin router "

    component.adapts( IOrder )

    interface.implements( IOriginRouter )

    def __init__( self, context ):
        self.context = context

    def getOrigin( self ):
        warehouses = self.getOrderWarehouses()
        if not warehouses:
            fallback_router = component.getAdapter( self.context, IOriginRouter, name="default" )
            return fallback_router.getOrigin()
        assert len(warehouses) == 1, "only one origin warehouse allowed per order, atm"
        warehouse = warehouses.pop()
        contact = self.getStoreContact()
        contact.name = warehouse.name
        return contact, warehouse.location

    def getOrderWarehouses( self ):
        items = filter( IShippableLineItem.providedBy, self.context.shopping_cart.values())
        if not items:
            return

        warehouse_names = set()
        for i in items:
            shippable = i.resolve()
            if shippable is None:
                continue
            inventory = interfaces.IProductInventory( shippable )
            warehouse_names.add( inventory.warehouse )

        container = component.getUtility( interfaces.IWarehouseContainer )
        return filter( None, map( container.get, warehouse_names ) )

    def getStoreContact( self ):
        store_settings = component.getUtility( IStoreSettings )
        contact = ContactInformation( name = ( store_settings.contact_company or store_settings.store_name ),
                                      phone_number = store_settings.contact_phone,
                                      email = store_settings.contact_email )
        return contact


from torchvision.datasets import CIFAR10, CIFAR100, MNIST, SVHN, FashionMNIST
from data.MVTecDataset import getMVTecDataset
from data.AdaptiveExposureDataset import getAdaptiveExposureDataset
import os
import torch
import numpy as np
import torchvision.transforms as transforms
from torch.utils.data import Dataset
from PIL import Image
from glob import glob
from utils import sparse2coarse
from data.constants import CIFAR10_PATH, CIFAR100_PATH, MNIST_PATH, FMNIST_PATH, SVHN_PATH, MVTEC_PATH, ADAPTIVE_PATH, mvtec_labels
import torchvision.transforms.functional as F
import requests
from PIL import Image
from tqdm import tqdm

dataset_paths = {
    "cifar10":CIFAR10_PATH,
    "cifar100":CIFAR100_PATH,
    "mnist":MNIST_PATH,
    "fashion":FMNIST_PATH,
    "svhn":SVHN_PATH,
    "mvtec":MVTEC_PATH,
    "adaptive":ADAPTIVE_PATH
}

tansform_224 = transforms.Compose([
                                    transforms.Resize(224),
                                    transforms.ToTensor()
                                ])

tansform_224_gray = transforms.Compose([
                                    transforms.Resize(224),
                                    transforms.Grayscale(num_output_channels=3),
                                    transforms.ToTensor()
                                ])

tansform_32 = transforms.Compose([
                                    transforms.Resize(32),
                                    transforms.ToTensor()
                                ])

tansform_32_gray = transforms.Compose([
                                    transforms.Resize(32),
                                    transforms.Grayscale(num_output_channels=3),
                                    transforms.ToTensor()
                                ])

class GeneralDataset(torch.utils.data.Dataset):
    def __init__(self, data, targets, transform=None):
        self.transform = transform
        self.data = [F.to_pil_image(x).convert('RGB') for x in data]
        self.targets = targets

    def __getitem__(self, index):
        image = self.data[index]
        target = self.targets[index]

        if self.transform is not None:
            image = self.transform(image)

        return image, target

    def __len__(self):
        return len(self.data)

class GeneralDatasetOODClf(torch.utils.data.Dataset):
    def __init__(self, normal_data, exposure_data, transform=None):
        self.transform = transform
        normal_data = [F.to_pil_image(x).convert('RGB') for x in normal_data]
        exposure_data = [F.to_pil_image(x).convert('RGB') for x in exposure_data]
        self.data = normal_data + exposure_data
        self.targets = [0] * len(normal_data) + [1] * len(exposure_data)

    def __getitem__(self, index):
        image = self.data[index]
        target = self.targets[index]

        if self.transform is not None:
            image = self.transform(image)

        return image, target

    def __len__(self):
        return len(self.data)

def get_dataloader(normal_dataset:str, normal_class_indx:int, exposure_dataset:str, batch_size):

    transform = None

    is_big: bool = True if normal_dataset in ['mvtec', 'ctscan'] else False
    is_colorful: bool = True if normal_dataset in ['cifar10', 'cifar100', 'svhn', 'mvtec'] else False

    if is_big:
        if is_colorful:
            transform = tansform_224
        else:
            transform = tansform_224_gray
    else:
        if is_colorful:
            transform = tansform_32
        else:
            transform = tansform_32_gray

    normal_data, test_data, test_targets = get_normal_class(dataset=normal_dataset, normal_class_indx=normal_class_indx, transform=transform)
    print("Length of Normal Dataset:", len(normal_data))
    print("Length of Test Dataset:", len(test_data))

    assert 1 in test_targets and 0 in test_targets

    exposure_data = get_exposure(dataset=exposure_dataset, normal_dataset=normal_dataset, normal_class_indx=normal_class_indx, count=len(normal_data))
    print("Length of Exposure Dataset:", len(exposure_data))

    trainset = GeneralDatasetOODClf(normal_data=normal_data, exposure_data=exposure_data, transform=transform)
    testset = GeneralDataset(data=test_data, targets = test_targets, transform=transform)
    del exposure_data, normal_data

    train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)
    test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2)

    return train_loader, test_loader

####################
#  Normal Datastes #
####################

def get_normal_class(dataset='cifar10', normal_class_indx = 0,  transform=None):
    datasets_builders = {
        "cifar10":CIFAR10,
        "cifar100":CIFAR100,
        "mnist":MNIST,
        "fashion":FashionMNIST,
        "svhn":SVHN,
        "mvtec":getMVTecDataset(normal_class_indx=normal_class_indx, only_anomaly_in_test=False)
    }

    if dataset in datasets_builders.keys():
        trainset = datasets_builders[dataset](root=dataset_paths[dataset], train=True, download=True) #CHECK transform ?

        if dataset == "cifar100":
            trainset.targets = sparse2coarse(trainset.targets)
        trainset.data = trainset.data[np.array(trainset.targets) == normal_class_indx]

        testset = datasets_builders[dataset](root=dataset_paths[dataset], train=False, download=True)
        if dataset == "cifar100":
            testset.targets = sparse2coarse(testset.targets)
        testset.targets  = [int(t!=normal_class_indx) for t in testset.targets]

        return [F.to_tensor(np.array(x).astype(np.uint8)) for x  in trainset.data], [F.to_tensor(np.array(x).astype(np.uint8)) for x  in testset.data], testset.targets
    else:
        raise Exception("Dataset is not supported yet. ")

######################
#  Exposure Datastes #
######################

def get_exposure(dataset:str='cifar10', normal_dataset:str='cifar100', normal_class_indx:int = 0, count:int = 0):
    assert count > 0

    datasets_builders = {
        "cifar10":CIFAR10,
        "cifar100":CIFAR100,
        "mnist":MNIST,
        "fashion":FashionMNIST,
        "svhn":SVHN,
        "mvtec":getMVTecDataset(
            normal_class_indx = None,
            only_anomaly_in_test = False if normal_dataset!='mvtec' else True,
            remove_class_indx = None if normal_dataset!='mvtec' else mvtec_labels[normal_class_indx]
        ),
        "adaptive":getAdaptiveExposureDataset(normal_dataset, normal_class_indx)
    }

    if dataset in datasets_builders.keys():
        exposure_train = datasets_builders[dataset](root=dataset_paths[dataset], train=True, download=True) #CHECK transforms ?

        assert len(exposure_train) > 0

        #exposure_test = datasets_builders[dataset](root=dataset_paths[dataset], train=False, download=True) #CHECK transforms ?
        if dataset == "cifar100":
            exposure_train.targets = sparse2coarse(exposure_train.targets)
            #exposure_test.targets = sparse2coarse(exposure_test.targets)

        if normal_dataset.lower() == dataset:
            exposure_train.data = exposure_train.data[np.array(exposure_train.targets) != normal_class_indx]
            #exposure_test.data = exposure_test.data[np.array(exposure_test.targets) != normal_class_indx]

        if dataset == 'adaptive':
            exposure_data = torch.stack(exposure_train.data)
        else:
            exposure_data = torch.tensor(exposure_train.data)

        del exposure_train

        #if exposure_data.size(0) < count:
        #    exposure_data = torch.cat((exposure_data, torch.tensor(exposure_test.data)), 0)
        #del exposure_train

        if exposure_data.size(0) < count:
            copy_dataset(exposure_data, count)

        indices = torch.randperm(exposure_data.size(0))[:count]
        exposure_data =  exposure_data[indices]

        if dataset == 'adaptive':
            return exposure_data
        else:
            return [F.to_tensor(np.array(x).astype(np.uint8)) for x  in exposure_data]
    else:
        raise Exception("Dataset is not supported yet. ")

def copy_dataset(dataset , target_count:int):
    while target_count > len(dataset):
        dataset = torch.cat((dataset, dataset.data), 0)

    return dataset

def get_ADAPTIVE_exposure(normal_dataset:str, normal_class_indx:int,count:int):
    exposure_data = []
    try:
        exposure_path = glob(os.path.join(ADAPTIVE_PATH, normal_dataset, f'{normal_class_indx}', "*.npy"), recursive=True)
        for path in exposure_path:
            exposure_data += np.load(path).tolist()
    except:
        raise ValueError('Wrong Exposure Address!')
        exit()

    exposure_data = torch.tensor(exposure_data)

    if exposure_data.size(0) < count:
        copy_dataset(exposure_data, count)

    indices = torch.randperm(exposure_data.size(0))[:count]
    exposure_data =  exposure_data[indices]

    return exposure_data


import logging
import time

import dataset
import discord
from discord import Member
from discord.ext import commands

from cogs.commands import settings
from utils import database, embeds

# Enabling logs
log = logging.getLogger(__name__)

class MutesHandler(commands.Cog):
    """Handles actions such as mute evasion."""

    def __init__(self, bot):
        self.bot = bot

    @commands.Cog.listener()
    async def on_member_remove(self, member: Member) -> None:
        # Open a connection to the database.
        db = dataset.connect(database.get_db())

        guild = member.guild
        mute_channel = discord.utils.get(guild.channels, name=f"mute-{member.id}")

        if mute_channel:
            mod_channel = guild.get_channel(settings.get_value("channel_moderation"))
            user = await self.bot.fetch_user(member.id)

            # Add an unmute entry in the database to prevent archive_mute_channel()'s unmuter throwing NoneType() exception.
            mute_entry = db["mod_logs"].find_one(user_id=member.id, type="mute")
            # Add an "unmute" entry into the database.
            if mute_entry:
                db["mod_logs"].insert(dict(
                    user_id=user.id,
                    mod_id=self.bot.user.id,
                    timestamp=int(time.time()),
                    reason="Mute evasion.",
                    type="unmute"
                ))

            # Update the mute entry in the timed_mod_actions database to prevent issues with task looping.
            tempmute_entry = db["timed_mod_actions"].find_one(user_id=member.id, is_done=False, action_type="mute")
            if tempmute_entry:
                db["timed_mod_actions"].update(dict(id=tempmute_entry["id"], is_done=True), ["id"])

            # Archive the mute channel
            mutes = self.bot.get_cog("MuteCog")
            await mutes.archive_mute_channel(
                ctx=None,
                user_id=user.id,
                reason="Mute channel archived after member banned due to mute evasion.",
                guild=guild,
            )

            # Add the ban to the mod_log database.
            db["mod_logs"].insert(dict(
                user_id=user.id,
                mod_id=self.bot.user.id,
                timestamp=int(time.time()),
                reason="Mute evasion.",
                type="ban"
            ))

            # Ban the user.
            await guild.ban(user, reason="Mute evasion.")

            # Creating the embed used to alert the moderators that the mute evading member was banned.
            embed = embeds.make_embed(
                ctx=None,
                title=f"Member {user.name}#{user.discriminator} banned",
                description=f"User {user.mention} was banned indefinitely because they evaded their timed mute by leaving.",
                thumbnail_url="https://i.imgur.com/l0jyxkz.png",
                color="soft_red"
            )
            await mod_channel.send(embed=embed)

        # Commit the changes to the database and close the connection.
        db.commit()
        db.close()

def setup(bot) -> None:
    """Load the cog."""
    bot.add_cog(MutesHandler(bot))
    log.info("Listener Loaded: mutes")


import logging
import psutil
import signal

from devil.android import device_errors
from devil.android import device_utils

def _KillWebServers():
  for s in [signal.SIGTERM, signal.SIGINT, signal.SIGQUIT, signal.SIGKILL]:
    signalled = []
    for server in ['lighttpd', 'webpagereplay']:
      for p in psutil.process_iter():
        try:
          if not server in ' '.join(p.cmdline):
            continue
          logging.info('Killing %s %s %s', s, server, p.pid)
          p.send_signal(s)
          signalled.append(p)
        except Exception: # pylint: disable=broad-except
          logging.exception('Failed killing %s %s', server, p.pid)
    for p in signalled:
      try:
        p.wait(1)
      except Exception: # pylint: disable=broad-except
        logging.exception('Failed waiting for %s to die.', p.pid)

def CleanupLeftoverProcesses(devices):
  """Clean up the test environment, restarting fresh adb and HTTP daemons.

  Args:
    devices: The devices to clean.
  """
  _KillWebServers()
  device_utils.RestartServer()

  def cleanup_device(d):
    try:
      d.WaitUntilFullyBooted()
      d.RestartAdbd()
      d.EnableRoot()
      d.WaitUntilFullyBooted()
    except (device_errors.CommandFailedError,
            device_errors.CommandTimeoutError):
      logging.exception('Failed to clean up device. Attempting to continue.')

  device_utils.DeviceUtils.parallel(devices).pMap(cleanup_device)


# -*- coding: utf-8 -*-
"""HTTP Endpoint for Netflix session management"""
from __future__ import absolute_import, division, unicode_literals

import json
import BaseHTTPServer

from SocketServer import TCPServer
import resources.lib.common as common

from .nfsession import NetflixSession

class NetflixHttpRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """Handles & translates requests from Inputstream to Netflix"""
    # pylint: disable=invalid-name, broad-except
    def do_HEAD(self):
        """Answers head requests with a success code"""
        self.send_response(200)

    def do_POST(self):
        """Loads the licence for the requested resource"""
        common.debug('Handling HTTP IPC call to {}'.format(self.path[1:]))
        func = getattr(NetflixSession, self.path[1:])
        length = int(self.headers.get('content-length', 0))
        data = json.loads(self.rfile.read(length)) or None
        result = func(self.server.netflix_session, data)
        self.send_response(200
                           if (not isinstance(result, dict) or
                               'error' not in result)
                           else 500)
        self.end_headers()
        self.wfile.write(json.dumps(result))
        self.finish()

    def log_message(self, *args):
        # pylint: disable=arguments-differ
        """Disable the BaseHTTPServer Log"""
        pass

class NetflixTCPServer(TCPServer):
    """Override TCPServer to allow usage of shared members"""
    def __init__(self, server_address):
        """Initialization of MSLTCPServer"""
        common.log('Constructing NetflixTCPServer')
        self.netflix_session = NetflixSession()
        TCPServer.__init__(self, server_address, NetflixHttpRequestHandler)


# -*- coding: utf-8 -*-

"""
@File    : balancedString.py
@Author  : wenhao
@Time    : 2023/2/13 9:26
@LC      : 1234
"""
from math import inf
from collections import Counter

class Solution:
    '''
    思路：
    同向双指针：
    '''
    def balancedString(self, s: str) -> int:
        cnt, m = Counter(s), len(s)
        if all(cnt[x] == m for x in "QWER"):
            return 0
        ans, left = inf, 0
        for right, c in enumerate(s):
            cnt[c] -= 1
            while all(cnt[x] <= m for x in "QWER"):
                ans = min(ans, right - left + 1)
                cnt[s[left]] += 1
                left += 1
        return ans


"""
Imagelab is the core class in CleanVision for finding all types of issues in an image dataset.
The methods in this module should suffice for most use-cases,
but advanced users can get extra flexibility via the code in other CleanVision modules.
"""
from __future__ import annotations

import random
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, TypeVar

import numpy as np
import pandas as pd
from PIL import Image

import cleanvision
from cleanvision.dataset.torch_dataset import TorchDataset
from cleanvision.dataset.utils import build_dataset
from cleanvision.issue_managers import (
    ISSUE_MANAGER_REGISTRY,
    IssueManagerFactory,
    IssueType,
)
from cleanvision.utils.base_issue_manager import IssueManager
from cleanvision.utils.constants import (
    DEFAULT_ISSUE_TYPES,
    DUPLICATE,
    DUPLICATE_ISSUE_TYPES_LIST,
    IMAGE_PROPERTY,
    IMAGE_PROPERTY_ISSUE_TYPES_LIST,
    SETS,
)
from cleanvision.utils.serialize import Serializer
from cleanvision.utils.utils import (
    deep_update_dict,
    get_is_issue_colname,
    get_max_n_jobs,
    get_score_colname,
    update_df,
)
from cleanvision.utils.viz_manager import VizManager

if TYPE_CHECKING:  # pragma: no cover
    import datasets
    from torchvision.datasets.vision import VisionDataset

__all__ = ["Imagelab"]
TImagelab = TypeVar("TImagelab", bound="Imagelab")

class Imagelab:
    """A single class to find all types of issues in image datasets.
    Imagelab detects issues in any image dataset and thus can be useful in most computer vision tasks including
    supervised and unsupervised training.
    Imagelab supports various formats for datasets: local folder containing images, a list of image
    filepaths, HuggingFace dataset and Torchvision dataset.
    Specify only one of these arguments: `data_path`, `filepaths`, (`hf_dataset`, `image_key`), `torchvision_dataset`

    Parameters
    ----------
    data_path : str
        Path to image files.
        Imagelab will recursively retrieve all image files from the specified path

    filepaths: List[str], optional
        Issue checks will be run on this list of image paths specified in `filepaths`.

    hf_dataset: datasets.Dataset
        Hugging Face dataset with images in PIL format accessible via some key in ``hf_dataset.features``.

    image_key: str
        Key used to access images within the Hugging Face `dataset.features` object. For many datasets, this key is just called "image".
        This argument must be specified if you provide a Hugging Face dataset; for other types of dataset this argument has no effect.

    torchvision_dataset: torchvision.datasets.vision.VisionDataset
        torchvision dataset where each individual  example is a tuple containing exactly one image in PIL format.

    Attributes
    ----------
    issues : pd.DataFrame
        Dataframe where each row corresponds to an image and columns specify which issues were detected in this image.
        It has two types of columns for each issue type:

        1. <issue_type>_score - This column contains a quality-score for each image for a particular type of issue.
        Scores are between 0 and 1, lower values indicate images exhibiting more severe instances of this issue.

        2. is_<issue_type>_issue - This column indicates whether or not the issue_type is detected in each image (a binary decision rather than numeric score).

    issue_summary : pd.DataFrame
        Dataframe where each row corresponds to a type of issue and columns summarize the overall prevalence of this issue in the dataset.
        Specifically, it shows the number of images detected with the issue.

    info : Dict
        Nested dictionary that contains statistics and other useful information about the dataset.
        Also contains additional information saved while checking for issues in the dataset.

    Raises
    ------
    ValueError
        If no images are found in the specified paths.
        If both `data_path` and `filepaths` are given or none of them are specified.

    Examples
    --------

    Basic usage of Imagelab class

    .. code-block:: python

        from cleanvision import Imagelab
        imagelab = Imagelab(data_path="FOLDER_WITH_IMAGES/")
        imagelab.find_issues()
        imagelab.report()

    """

    def __init__(
        self,
        data_path: Optional[str] = None,
        filepaths: Optional[List[str]] = None,
        hf_dataset: Optional["datasets.Dataset"] = None,
        image_key: Optional[str] = None,
        torchvision_dataset: Optional["VisionDataset"] = None,
        storage_opts: Dict[str, Any] = {},
    ) -> None:
        self._dataset = build_dataset(
            data_path,
            filepaths,
            hf_dataset,
            image_key,
            torchvision_dataset,
            storage_opts=storage_opts,
        )
        if len(self._dataset) == 0:
            raise ValueError("No images found in the dataset specified")
        self.info: Dict[str, Any] = {"statistics": {}}
        self.issue_summary: pd.DataFrame = pd.DataFrame(
            columns=["issue_type", "num_images"]
        ).astype({"issue_type": str, "num_images": np.int64})

        self.issues: pd.DataFrame = pd.DataFrame(index=self._dataset.index)
        self._issue_types: List[str] = []
        self._issue_managers: Dict[str, IssueManager] = {}

        # TODO: can be loaded from a file later
        self._config: Dict[str, Any] = self._set_default_config()
        self.cleanvision_version: str = cleanvision.__version__

    def _set_default_config(self) -> Dict[str, Any]:
        """Sets default values for various config variables used in Imagelab class
        The naming convention for methods is {method_name}_{config_variable_name}

        Returns
        -------
        Dict[str, Any]
            Returns a dict with keys as config variables and their values as dict values
        """
        return {
            "visualize_num_images_per_row": 4,
            "report_num_images": 4,
            "report_max_prevalence": 0.5,
            "report_cell_size": (2, 2),
        }

    @staticmethod
    def list_default_issue_types() -> List[str]:
        """Returns a list of the issue types that are run by default in :py:meth:`Imagelab.find_issues`"""
        return DEFAULT_ISSUE_TYPES

    @staticmethod
    def list_possible_issue_types() -> List[str]:
        """Returns a list of all the possible issue types that can be run in :py:meth:`Imagelab.find_issues`
        This list will also include custom issue types if properly added.
        """
        issue_types = Imagelab.list_default_issue_types()
        for key in ISSUE_MANAGER_REGISTRY:
            if key not in [IMAGE_PROPERTY, DUPLICATE]:
                issue_types.append(key)
        return list(set(issue_types))

    def _get_issues_to_compute(
        self, issue_types_with_params: Optional[Dict[str, Any]]
    ) -> Dict[str, Any]:
        if not issue_types_with_params:
            to_compute_issues_with_params: Dict[str, Any] = {
                issue_type: {} for issue_type in self.list_default_issue_types()
            }
        else:
            to_compute_issues_with_params = {
                issue_type_str: params
                for issue_type_str, params in issue_types_with_params.items()
            }
        return to_compute_issues_with_params

    def find_issues(
        self,
        issue_types: Optional[Dict[str, Any]] = None,
        n_jobs: Optional[int] = None,
        verbose: bool = True,
    ) -> None:
        """Finds issues in the dataset.
        If `issue_types` is not provided, dataset is checked for a default set of issue types.
        To see default set: :py:meth:`Imagelab.list_default_issue_types`

        Parameters
        ----------
        issue_types : Dict[str, Any], optional
            Dict with issue types to check as keys.
            The value of this dict is a dict containing hyperparameters for each issue type.
        n_jobs :  int, default=None
            Number of processing threads used by multiprocessing.
            Default None sets to the number of cores on your CPU (physical cores if you have psutil package installed, otherwise logical cores).
            Set this to 1 to disable parallel processing (if its causing issues). Windows users may see a speed-up with n_jobs=1.
        verbose : bool, default=True
            If True, prints helpful information while checking for issues.

        Examples
        --------
        To check for all default issue types use

        .. code-block:: python

            imagelab.find_issues()

        To check for specific issue types with default settings

        .. code-block:: python

            issue_types = {
                "dark": {},
                "blurry": {}
            }
            imagelab.find_issues(issue_types)

        To check for issue types with different hyperparameters. Different issue types can have different hyperparameters.

        .. code-block:: python

            issue_types = {
                "dark": {"threshold": 0.1},
                "blurry": {}
            }
            imagelab.find_issues(issue_types)

        """
        to_compute_issues_with_params = self._get_issues_to_compute(issue_types)
        if verbose:
            print(
                f"Checking for {', '.join([issue_type for issue_type in to_compute_issues_with_params.keys()])} images ..."
            )

        # update issue_types
        self._issue_types = list(
            set(self._issue_types).union(set(to_compute_issues_with_params.keys()))
        )

        # set issue managers
        issue_type_groups = self._get_issue_type_groups(to_compute_issues_with_params)
        self._set_issue_managers(issue_type_groups)

        # set number of jobs for parallelizing computation
        if n_jobs is None:
            n_jobs = get_max_n_jobs()
            if isinstance(self._dataset, TorchDataset):
                n_jobs = 1

        # find issues
        for issue_type_group, params in issue_type_groups.items():
            issue_manager = self._issue_managers[issue_type_group]
            issue_manager.find_issues(
                params=params,
                dataset=self._dataset,
                imagelab_info=self.info,
                n_jobs=n_jobs,
            )

            # update issues, issue_summary and info
            self._update_issues(issue_manager.issues)
            self._update_issue_summary(issue_manager.summary)
            self._update_info(issue_manager.info)

        self.issue_summary = self.issue_summary.sort_values(
            by=["num_images"], ascending=False
        )
        self.issue_summary = self.issue_summary.reset_index(drop=True)
        if verbose:
            print(
                f"Issue checks completed. {self.issue_summary['num_images'].sum()} issues found in the dataset. To see a detailed report of issues found, use imagelab.report()."
            )
        return

    def _update_info(self, issue_manager_info: Dict[str, Any]) -> None:
        deep_update_dict(self.info, issue_manager_info)

    def _update_issue_summary(self, issue_manager_summary: pd.DataFrame) -> None:
        # Remove results for issue types computed again
        self.issue_summary = self.issue_summary[
            ~self.issue_summary["issue_type"].isin(issue_manager_summary["issue_type"])
        ]
        # concat new results
        self.issue_summary = pd.concat(
            [self.issue_summary, issue_manager_summary], axis=0, ignore_index=True
        )

    def _update_issues(self, issue_manager_issues: pd.DataFrame) -> None:
        self.issues = update_df(self.issues, issue_manager_issues)

    def _get_issue_type_groups(
        self, issue_types_with_params: Dict[str, Any]
    ) -> Dict[str, Any]:
        issue_type_groups = {}

        for issue_type, params in issue_types_with_params.items():
            group_name = None
            if issue_type in IMAGE_PROPERTY_ISSUE_TYPES_LIST:
                group_name = IMAGE_PROPERTY
            elif issue_type in DUPLICATE_ISSUE_TYPES_LIST:
                group_name = DUPLICATE
            else:
                issue_type_groups[issue_type] = params

            if group_name:
                if issue_type_groups.get(group_name):
                    issue_type_groups[group_name][issue_type] = params
                else:
                    issue_type_groups[group_name] = {issue_type: params}
        return issue_type_groups

    def _set_issue_managers(self, issue_type_groups: Dict[str, Any]) -> None:
        for issue_type_group, params in issue_type_groups.items():
            self._issue_managers[issue_type_group] = IssueManagerFactory.from_str(
                issue_type_group
            )()

    def _filter_report(
        self, issue_types: List[str], max_prevalence: float
    ) -> List[str]:
        """Filters issues by max_prevalence in the dataset."""
        issue_summary = self.issue_summary[
            self.issue_summary["issue_type"].isin(issue_types)
        ]
        issue_to_report = []
        for row in issue_summary.itertuples(index=False):
            if getattr(row, "num_images") / len(self._dataset) < max_prevalence:
                issue_to_report.append(getattr(row, "issue_type"))
            else:
                print(
                    f"Removing {getattr(row, 'issue_type')} from potential issues in the dataset as it exceeds "
                    f"max_prevalence={max_prevalence} "
                )
        return issue_to_report

    def _get_report_args(self, user_supplied_args: Dict[str, Any]) -> Dict[str, Any]:
        report_args = {
            "max_prevalence": self._config["report_max_prevalence"],
            "num_images": self._config["report_num_images"],
            "cell_size": self._config["report_cell_size"],
        }

        non_none_args = {
            arg: value for arg, value in user_supplied_args.items() if value is not None
        }

        return {**report_args, **non_none_args}

    def report(
        self,
        issue_types: Optional[List[str]] = None,
        max_prevalence: Optional[float] = None,
        num_images: Optional[int] = None,
        verbosity: int = 1,
        print_summary: bool = True,
        show_id: bool = False,
    ) -> None:
        """Prints summary of the issues found in your dataset.
        By default, this method depicts the images representing top-most severe instances of each issue type.

        Parameters
        ----------
        issue_types : List[str], optional
            List of issue types to consider in report.
            This must be subset of the issue types specified in :py:meth:`Imagelab.find_issues``.

        max_prevalence : float, default=0.5
            Value between 0 and 1
            Issue types that are detected in more than `max_prevalence` fraction of the images in dataset will be omitted from the report.
            You are presumably already aware of these in your dataset.

        num_images : int, default=4
            Maximum number of images to show for issue type reported. These are examples of the top-most severe instances of the issue in your dataset.

        verbosity : int, {1, 2, 3, 4}
            Increasing verbosity increases the detail of the report. Set this to 1 to report less information, or to 4 to report the most information.

        print_summary : bool, default=True
            If True, prints the summary of issues found in the dataset.

        show_id: bool, default=False
            If True, prints the dataset ID of each image shown in the report.

        Examples
        --------
        Default usage

        .. code-block:: python

            imagelab.report()

        Report specific issue types

        .. code-block:: python

            issue_types = ["dark", "near_duplicates"]
            imagelab.report(issue_types=issue_types)

        """
        assert isinstance(verbosity, int) and 0 <= verbosity < 5

        user_supplied_args = locals()
        report_args = self._get_report_args(user_supplied_args)

        issue_types_to_report = (
            issue_types if issue_types else self.issue_summary["issue_type"].tolist()
        )

        # filter issues based on max_prevalence in the dataset
        filtered_issue_types = self._filter_report(
            issue_types_to_report, report_args["max_prevalence"]
        )

        issue_summary = self.issue_summary[
            self.issue_summary["issue_type"].isin(filtered_issue_types)
        ]
        if len(issue_summary) > 0:
            if verbosity:
                print("Issues found in images in order of severity in the dataset\n")
            if print_summary:
                self._pprint_issue_summary(issue_summary)
            for issue_type in filtered_issue_types:
                if (
                    self.issue_summary.query(f"issue_type == '{issue_type}'")[
                        "num_images"
                    ].values[0]
                    == 0
                ):
                    continue
                print(f"{' ' + issue_type + ' images ':-^60}\n")
                print(
                    f"Number of examples with this issue: {self.issues[get_is_issue_colname(issue_type)].sum()}\n"
                    f"Examples representing most severe instances of this issue:\n"
                )
                self._visualize(
                    issue_type,
                    report_args["num_images"],
                    report_args["cell_size"],
                    show_id,
                )
        else:
            print(
                "Please specify some issue_types to check for in imagelab.find_issues()."
            )

    def _pprint_issue_summary(self, issue_summary: pd.DataFrame) -> None:
        issue_summary_copy = issue_summary.copy()
        issue_summary_copy.dropna(axis=1, how="all", inplace=True)
        issue_summary_copy.fillna("N/A", inplace=True)
        print(issue_summary_copy.to_markdown(), "\n")

    def _get_issue_manager(self, issue_type_str: str) -> IssueManager:
        if issue_type_str in IMAGE_PROPERTY_ISSUE_TYPES_LIST:
            return self._issue_managers[IMAGE_PROPERTY]
        elif issue_type_str in DUPLICATE_ISSUE_TYPES_LIST:
            return self._issue_managers[DUPLICATE]
        else:
            return self._issue_managers[issue_type_str]

    def _visualize(
        self,
        issue_type: str,
        num_images: int,
        cell_size: Tuple[int, int],
        show_id: bool,
    ) -> None:
        # todo: remove dependency on issue manager
        issue_manager = self._get_issue_manager(issue_type)
        viz_name = issue_manager.visualization

        if viz_name == "individual_images":
            sorted_df = self.issues.sort_values(by=get_score_colname(issue_type))
            sorted_df = sorted_df[sorted_df[get_is_issue_colname(issue_type)] == 1]

            scores = sorted_df.head(num_images)[get_score_colname(issue_type)]
            indices = scores.index.tolist()
            images = [self._dataset[i] for i in indices]

            # construct title info
            title_info = {"scores": [f"score : {x:.4f}" for x in scores]}
            if show_id:
                title_info["ids"] = [f"id : {i}" for i in indices]
            if issue_type == IssueType.ODD_SIZE.value:
                title_info["size"] = [
                    f"original size: {image.size}" for image in images
                ]

            if images:
                VizManager.individual_images(
                    images=images,
                    title_info=title_info,
                    ncols=self._config["visualize_num_images_per_row"],
                    cell_size=cell_size,
                )

        elif viz_name == "image_sets":
            image_sets_indices = sorted(
                self.info[issue_type][SETS], key=len, reverse=True
            )
            image_sets_indices = image_sets_indices[:num_images]
            image_sets = []
            for indices in image_sets_indices:
                image_sets.append([self._dataset[index] for index in indices])

            title_info_sets = []
            for s in image_sets_indices:
                title_info = {"name": [self._dataset.get_name(index) for index in s]}
                title_info_sets.append(title_info)

            if image_sets:
                VizManager.image_sets(
                    image_sets,
                    title_info_sets,
                    ncols=self._config["visualize_num_images_per_row"],
                    cell_size=cell_size,
                )

    # todo: compress this code
    def visualize(
        self,
        image_files: Optional[List[str]] = None,
        indices: Optional[List[str | int]] = None,
        issue_types: Optional[List[str]] = None,
        num_images: int = 4,
        cell_size: Tuple[int, int] = (2, 2),
        show_id: bool = False,
    ) -> None:
        """Show specific images.

        Can be used for visualizing either:
        1. Particular images with paths given in `image_files`.
        2. Images representing top-most severe instances of given `issue_types` detected the dataset.
        3. If no `image_files` or `issue_types` are given, random images will be shown from the dataset.

        If `image_files` is given, this overrides the argument `issue_types`.

        Parameters
        ----------

        image_files : List[str], optional
            List of filepaths for images to visualize.

        indices: List[str|int], optional
            List of indices of images in the dataset to visualize.
            If the dataset is a local data_path, the indices are filepaths, which is also the index in `imagelab.issues` dataframe.
            If the dataset is a huggingface or torchvision dataset, indices are of type int and corresponding to the indices in the dataset object.

        issue_types: List[str], optional
            List of issue types to visualize. For each type of issue, will show a few images representing the top-most severe instances of this issue in the dataset.

        num_images : int, optional
            Number of images to visualize from the dataset.
            These images are randomly selected if `issue_types` is ``None``.
            If `issue_types` is given, then this is the number of images for each issue type to visualize
            (images representing top-most severe instances of this issue will be shown).
            If `image_files` is given, this argument is ignored.

        cell_size : Tuple[int, int], optional
            Dimensions controlling the size of each image in the depicted image grid.

        Examples
        --------

        To visualize random images from the dataset

        .. code-block:: python

            imagelab.visualize()

        .. code-block:: python

            imagelab.visualize(num_images=8)

        To visualize specific images from the dataset

        .. code-block:: python

            image_files = ["./dataset/cat.png", "./dataset/dog.png", "./dataset/mouse.png"]
            imagelab.visualize(image_files=image_files)

        To visualize top examples of specific issue types from the dataset

        .. code-block:: python

            issue_types = ["dark", "odd_aspect_ratio"]
            imagelab.visualize(issue_types=issue_types)

        """
        if issue_types is not None:
            if len(issue_types) == 0:
                raise ValueError("issue_types list is empty")
            for issue_type in issue_types:
                self._visualize(issue_type, num_images, cell_size, show_id)
        elif image_files is not None:
            if len(image_files) == 0:
                raise ValueError("image_files list is empty.")
            images = [Image.open(path) for path in image_files]
            title_info = {"path": [path.split("/")[-1] for path in image_files]}
            VizManager.individual_images(
                images,
                title_info,
                ncols=self._config["visualize_num_images_per_row"],
                cell_size=cell_size,
            )
        elif indices:
            images = [self._dataset[i] for i in indices]
            title_info = {"name": [self._dataset.get_name(i) for i in indices]}
            VizManager.individual_images(
                images,
                title_info,
                ncols=self._config["visualize_num_images_per_row"],
                cell_size=cell_size,
            )
        else:
            print("Sample images from the dataset")

            if image_files is None:
                image_indices = random.sample(
                    self._dataset.index, min(num_images, len(self._dataset))
                )
                images = [self._dataset[i] for i in image_indices]
                title_info = {
                    "name": [self._dataset.get_name(i) for i in image_indices]
                }
                VizManager.individual_images(
                    images,
                    title_info,
                    ncols=self._config["visualize_num_images_per_row"],
                    cell_size=cell_size,
                )

    # Todo: Improve mypy dict typechecking so this does not return any
    def get_stats(self) -> Any:
        """Returns dict of statistics computed from images when auditing the data such as: brightness, color space, aspect ratio, etc.
        If statistics have not been computed yet, then returns ``None``.
        """
        return self.info["statistics"]

    def save(self, path: str, force: bool = False) -> None:
        """Saves this Imagelab instance, :py:attr:`issues` and :py:attr:`issue_summary` into a folder at the given path.
        Your saved Imagelab should be loaded from the same version of the CleanVision package to avoid inconsistencies.
        This method does not save your image files.

        Parameters
        ----------
        path : str
            Path to folder where this Imagelab instance will be saved on disk.

        force: bool, default=False
            If set to True, any existing files at `path` will be overwritten.
        """
        Serializer.serialize(path=path, imagelab=self, force=force)

    @classmethod
    def load(
        cls: Type[TImagelab], path: str, data_path: Optional[str] = None
    ) -> Imagelab:
        """Loads Imagelab from given path.

        Parameters
        ----------
        path : str
            Path to the saved Imagelab folder previously specified in :py:meth:`Imagelab.save` (not the individual pickle file).
        data_path : str
            Path to image dataset previously used in Imagelab, if your data exists locally as images in a folder.
            If the `data_path` is changed, the code will break as Imagelab functionalities are dependent on it.
            You should be using the same version of the CleanVision package previously used when saving Imagelab.

        Returns
        -------
        Imagelab
            Returns a saved instance of Imagelab
        """
        imagelab = Serializer.deserialize(path)
        return imagelab


from ..binary import BinaryReader, BinaryWriter
from ..binary.types import Int8, Int16, Int32, UInt8, UInt16, UInt32

class HKHeader:
    """Represents Havok file header
    """

    # fmt:off
    magic0: UInt32 = 0x57E0E057  # 0x00  # Always 0x57E0E057
    magic1: UInt32 = 0x10C0C010  # 0x04  # Always 0x10C0C010
    user_tag: Int32 = 0x0  # 0x08  # Always 0x00 (?)
    version: Int32 = 0x0B  # 0x0C  # Always 0x0B

    pointer_size: Int8  # 0x10  # 4 on WiiU, 8 on Switch
    endian: Int8  # 0x11  # 0 (big) on WiiU, 1 (little) on Switch
    padding_option: Int8  # 0x12  # 0 on WiiU, 1 on Switch
    base_class: Int8  # 0x13  # Always 1

    section_count: Int32 = 3  # 0x14  # Always 3 (classnames, types, data)
    data_section_id: Int32 = 2  # 0x18  # Always 2, refers to __data__ section
    data_section_offset: UInt32 = 0  # 0x1C  # Always 0, beginning offset in __data__
    classnames_section_id: Int32 = 0  # 0x20  # Always 0, refers to __classnames__ section
    classnames_section_offset: UInt32 = 0x4B  # 0x24  # Always 75, offset to hkRootLevelContainer?
    hk_version: str = "hk_2014.2.0-r1"  # 0x28  # Always same, terminated with 0xFF
    flags: Int32 = 0  # 0x38  # Always 0x00 (?)
    max_predicate: Int16 = 21  # 0x3C  # Always 0x15 (?)
    predicate_size_plus_padding: UInt16 = 0  # 0x3E  # 0 or 16; additional data (?)

    unk40: Int16 = 20  # 0x40  # Always 20
    unk42: Int16 = 0  # 0x42  # Always 0 (padding?)
    unk44: Int32 = 0  # 0x44  # Always 0 (padding?)
    unk48: Int32 = 0  # 0x48  # Always 0 (padding?)
    unk4C: Int32 = 0  # 0x4C  # Always 0 (padding?)

    # fmt:on

    def to_switch(self):
        self.pointer_size = 8
        self.endian = 1
        self.padding_option = 1
        self.base_class = 1

    def to_wiiu(self):
        self.pointer_size = 4
        self.endian = 0
        self.padding_option = 0
        self.base_class = 1

    def read(self, br: BinaryReader):
        # Read the header
        self.magic0 = br.assert_uint32(0x57E0E057)
        self.magic1 = br.assert_uint32(0x10C0C010)
        self.user_tag = br.assert_int32(0x0)
        self.version = br.assert_int32(0x0B)

        self.pointer_size = br.assert_int8(4, 8)
        self.endian = br.assert_int8(0, 1)
        self.padding_option = br.assert_int8(0, 1)
        self.base_class = br.assert_int8(1)

        self.section_count = br.assert_int32(3)
        self.data_section_id = br.assert_int32(2)
        self.data_section_offset = br.assert_uint32(0x0)

        self.classnames_section_id = br.assert_int32(0)
        self.classnames_section_offset = br.assert_uint32(0x4B)
        self.hk_version = br.assert_string("hk_2014.2.0-r1", size=15)
        br.assert_uint8(0xFF)

        self.flags = br.assert_int32(0x0)
        self.max_predicate = br.assert_int16(0x15)
        self.predicate_size_plus_padding = br.assert_uint16(0, 16)

        if self.predicate_size_plus_padding == 0:
            pass
        elif self.predicate_size_plus_padding == 16:
            self.unk40 = br.assert_int16(0x14)
            self.unk42 = br.assert_int16(0x0)
            self.unk44 = br.assert_int32(0x0)
            self.unk48 = br.assert_int32(0x0)
            self.unk4C = br.assert_int32(0x0)
        else:
            raise NotImplementedError("Invalid predicate size? Idk really")

    def write(self, bw: BinaryWriter):
        bw.write_uint32(self.magic0)
        bw.write_uint32(self.magic1)
        bw.write_int32(self.user_tag)
        bw.write_int32(self.version)

        bw.write_int8(self.pointer_size)
        bw.write_int8(self.endian)
        bw.write_int8(self.padding_option)
        bw.write_int8(self.base_class)
        bw.write_int32(self.section_count)
        bw.write_int32(self.data_section_id)
        bw.write_uint32(self.data_section_offset)

        bw.write_int32(self.classnames_section_id)
        bw.write_uint32(self.classnames_section_offset)
        bw.write_string(self.hk_version, size=15)
        bw.write_uint8(UInt8(0xFF))

        bw.write_int32(self.flags)
        bw.write_int16(self.max_predicate)
        bw.write_uint16(self.predicate_size_plus_padding)

        if self.predicate_size_plus_padding == 16:
            bw.write_int16(self.unk40)
            bw.write_int16(self.unk42)
            bw.write_int32(self.unk44)
            bw.write_int32(self.unk48)
            bw.write_int32(self.unk4C)

    def as_dict(self):
        if self.predicate_size_plus_padding == 16:
            return {
                "unk40": self.unk40,
                "unk42": self.unk42,
                "unk44": self.unk44,
                "unk48": self.unk48,
                "unk4C": self.unk4C,
            }
        elif self.predicate_size_plus_padding == 0:
            return None
        else:
            raise NotImplementedError()

    @classmethod
    def from_dict(cls, d: dict):
        inst = cls()
        if d:
            inst.predicate_size_plus_padding = UInt16(16)
            inst.unk40, inst.unk42, inst.unk44, inst.unk48, inst.unk4C = d.values()

        return inst

    def __repr__(self):
        return f"{self.__class__.__name__}({self.pointer_size}, {self.endian}, {self.padding_option}, {self.base_class}, {self.predicate_size_plus_padding})"


import cv2
import argparse

ap = argparse.ArgumentParser()
ap.add_argument('-i' ,'--image' ,required=True ,help = 'path to the input image')
args = vars(ap.parse_args())

image = cv2.imread(args['image'])
gray = cv2.cvtColor(image ,cv2.COLOR_BGR2GRAY)
blurred = cv2.GaussianBlur(gray ,(5 ,5) ,0)
#last argument in blurring is corresponding to the standard deviation along x and y direction of
#filter. if not provided separatelty both are taken as 0.
retr ,thresh = cv2.threshold(blurred ,60 ,255 ,cv2.THRESH_BINARY)

cntrs ,heirarchy = cv2.findContours(thresh.copy() ,cv2.RETR_EXTERNAL ,cv2.CHAIN_APPROX_SIMPLE)

for c in cntrs:

    M = cv2.moments(c)
    cX = int(M['m10'] / M['m00'])
    cY = int(M['m01'] / M['m00'])
    cv2.drawContours(image ,[c] ,-1 ,(0 ,0 ,255) ,2)
    cv2.circle(image ,(cX ,cY) ,7 ,(255,255,255) ,-1)
    cv2.putText(image ,"center" ,(cX -10 ,cY -10) ,cv2.FONT_HERSHEY_TRIPLEX ,0.5 ,(255 ,255 ,255) ,2)

    cv2.imshow('center drawn' ,image)
    cv2.waitKey(0)


# pygame python第三方游戏库   .pyd 动态模块(可以导入,但是看不到源码) .py 静态模块
import pygame  # 官方推荐这样导入
from pygame.locals import *
import sys
import time
import random

# 定义常量记录数据 (常量特点: 字母全大写  一旦定义,不要修改记录的值)
WINDOW_H = 768
WINDOW_W = 512

class Bullet:  # 子弹类
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 子弹图片
        self.x = x  # 子弹坐标
        self.y = y
        self.window = window  # 子弹所在的窗口

    def display(self):
        """贴子弹图"""
        self.window.blit(self.img, (self.x, self.y))

    def move(self):
        """向上飞"""
        self.y -= 15

    # def __del__(self):
    #     print("子弹将要被删除")

class EnemyPlane:  # 敌人飞机
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 飞机图片
        self.x = x  # 飞机坐标
        self.y = y
        self.window = window  # 飞机所在的窗口

    def display(self):
        """贴飞机图"""
        self.window.blit(self.img, (self.x, self.y))

    def move(self):
        """向下飞"""
        self.y += 10
        # 判断是否飞出边界
        if self.y >= WINDOW_H:  # 重新回到窗口顶部
            self.y = random.randint(-500, -70)
            self.x = random.randint(0, WINDOW_W - 100)
            self.img = pygame.image.load("res/img-plane_%d.png" % random.randint(1, 7))

class HeroPlane:  # 英雄飞机类
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 飞机图片
        self.x = x  # 飞机坐标
        self.y = y
        self.window = window  # 飞机所在的窗口
        self.bullets = []  # 记录发出的所有子弹

    def display(self):
        """贴飞机图"""
        self.window.blit(self.img, (self.x, self.y))  # 290 500

    def move_left(self):
        """左移"""
        self.x -= 5

    def move_right(self):
        """右移"""
        self.x += 5

    def fire(self):
        """发射子弹"""
        # 创建子弹对象
        # 子弹x = 飞机x + 飞机宽度一半 - 子弹宽度一半
        # 子弹y =  飞机y - 子弹高度
        bullet = Bullet("res/bullet_9.png", self.x + 50, self.y - 31, self.window)
        self.bullets.append(bullet)

    def display_bullets(self):
        """贴子弹图"""
        out_window_bullets = []
        for bullet in self.bullets:
            # 判断子弹是否飞出边界
            if bullet.y >= -31:  # 在边界内,显示和移动
                bullet.display()
                bullet.move()
            else:  # 出了边界,删除子弹
                # self.bullets.remove(bullet)
                out_window_bullets.append(bullet)

        for out_window_bullet in out_window_bullets:
            self.bullets.remove(out_window_bullet)

class Map:  # 地图类
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 地图图片
        self.x = x  # 地图坐标
        self.y = y
        self.window = window  # 地图所在的窗口

    def display(self):
        """贴地图"""
        self.window.blit(self.img, (self.x, self.y))  # 290 500

def main():  # 一般将程序的入口定义为main函数
    """主函数"""
    # 1. 创建窗口
    window = pygame.display.set_mode((WINDOW_W, WINDOW_H))
    # 加载图片
    # bg_img = pygame.image.load("res/img_bg_level_1.jpg")
    # 创建地图对象
    map = Map("res/img_bg_level_1.jpg", 0, 0, window)
    # 创建飞机对象
    hero_plane = HeroPlane("res/hero2.png", WINDOW_W / 2 - 60, WINDOW_H / 2 - 39, window)
    enemy_list = []  # 记录每架敌机
    for _ in range(5):
        enemy_plane = EnemyPlane('res/img-plane_%d.png' % random.randint(1, 7), random.randint(0, WINDOW_W-100), random.randint(-500, -70), window)
        enemy_list.append(enemy_plane)
    while True:
        # 贴背景图
        # window.blit(bg_img, (0, 0))
        map.display()
        # 贴飞机图
        hero_plane.display()
        for enemy in enemy_list:
            enemy.display()
            enemy.move()
        # 贴子弹图
        hero_plane.display_bullets()
        # 3.刷新窗口
        pygame.display.update()

        # 获取新事件
        for event in pygame.event.get():
            # 1. 鼠标点击关闭窗口事件
            if event.type == QUIT:
                print("点击关闭窗口按钮")
                sys.exit()  # 关闭程序
            # 2. 键盘按下事件
            if event.type == KEYDOWN:
                # 判断用户按键
                if event.key == K_SPACE:
                    hero_plane.fire()

        # 检测键盘长按事件
        pressed_keys = pygame.key.get_pressed()
        if pressed_keys[K_a] or pressed_keys[K_LEFT]:
            # x -= 5
            hero_plane.move_left()
        if pressed_keys[K_d] or pressed_keys[K_RIGHT]:
            # x += 5
            hero_plane.move_right()
        time.sleep(0.02)  # 让循环每执行一次,休息一段时间  让CPU可以做别的工作

if __name__ == '__main__':
    main()


import matplotlib.pyplot as plt
import plotly.graph_objs as go
from plotly.offline import plot
from plotly.subplots import make_subplots

def plot_clandlestics(df):

    fig = go.Figure(data=[go.Candlestick(x=df['date'],
                                     open=df['open'],
                                     high=df['high'],
                                     low=df['low'],
                                     close=df['close']),
                      go.Scatter(x=df['date'], y=df['MA_close_21'], line=dict(color='orange', width=2)),
                      go.Scatter(x=df['date'], y=df['MA_close_100'], line=dict(color='green', width=2))])

    fig.update_layout(xaxis_rangeslider_visible=False)

    plot(fig, filename='go_candle1.html')

def plot_cross_ma(df):
    plt.figure(figsize=(15,5))
    plt.plot(df['close'], label='Close')
    plt.plot(df['MA9'], label='MA9')
    plt.plot(df['MA21'], label='MA21')

    plt.plot(df[df['PositionCrossMA'] == 1].index,
            df['MA20'][df['PositionCrossMA'] == 1],
            '^', markersize = 15, color = 'g', label = 'buy')

    plt.plot(df[df['PositionCrossMA'] == -1].index,
            df['MA20'][df['PositionCrossMA'] == -1],
            'v', markersize = 15, color = 'r', label = 'sell')
    plt.title('Price chart ')
    plt.show()
    return None

def plot_price(df, attr):
    # plot price
    plt.figure(figsize=(15,5))
    plt.plot(df[attr])
    plt.title('Price chart ' + attr)
    plt.show()
    return None

def plot_RSI(df, upper=80, down=20):
    # plot correspondingRSI values and significant levels
    plt.figure(figsize=(15,5))
    plt.title('RSI chart')
    plt.plot(df['RSI'])

    plt.axhline(0, linestyle='--', alpha=0.1)
    plt.axhline(down, linestyle='--', alpha=0.5)
    plt.axhline(30, linestyle='--')

    plt.axhline(70, linestyle='--')
    plt.axhline(upper, linestyle='--', alpha=0.5)
    plt.axhline(100, linestyle='--', alpha=0.1)
    plt.show()
    return None

def plot_stoch_RSI(df, upper=80, down=20):
    # plot corresponding Stoch RSI values and significant levels
    plt.figure(figsize=(15,5))
    plt.title('stochRSI chart')
    plt.plot(df['K'])
    plt.plot(df['D'])

    plt.axhline(0, linestyle='--', alpha=0.1)
    plt.axhline(down, linestyle='--', alpha=0.5)
    #plt.axhline(30, linestyle='--')

    #plt.axhline(70, linestyle='--')
    plt.axhline(80, linestyle='--', alpha=0.5)
    plt.axhline(upper, linestyle='--', alpha=0.1)
    plt.show()
    return None


"""SmartDevice URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.conf.urls import url
from django.urls import include, path
from rest_framework import routers
from sensors import views

router = routers.DefaultRouter()
router.register(r'users', views.UserViewSet)
router.register(r'rooms', views.RoomViewSet)
router.register(r'groups', views.GroupViewSet)

urlpatterns = [
    path('', include(router.urls)),
    path('rooms/', views.room_list),
    path('room/<int:pk>', views.room_detail),
    path('device_scan/<str:device_id>', views.device_scan),
    path('register_user/', views.register_user),
    path('camera_record/', views.face_record),
    path('sensor_stream/<str:device_id>', views.sensor_data_stream),
    path('fetch_by/', views.fetch_sensor_data_by),
    path('sec_sensor_data/', views.sec_sensor_data),
    path('get_sensor_data/', views.get_sensor_data),
    path('get_location_data/', views.fetch_location_data_by),
    path('admin/', admin.site.urls),
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),

    # query apis
    path('room/', views.room),
    path('room_info/', views.room_info),
    path('people_building/', views.people_building),
    path('people_room/', views.people_room),
    path('person_room/', views.person_room),
    path('utilization/', views.utilization)
]


from keras.preprocessing.image import ImageDataGenerator
from keras.applications.inception_v3 import InceptionV3
from keras import backend as K
K.set_image_dim_ordering('th')

import numpy as np
import os

#Здесь делаем аугментацию.
# Для этого в Keras предусмотрены так называемые ImageDataGenerator.
# Они будут брать изображения прямо из папок и проводить над ними все необходимые трансформации.

#Картинки каждого класса должны быть в отдельных папках. Для того, чтобы нам не загружать оперативную память изображениями,
# мы их трансформируем прямо перед загрузкой в сеть. Для этого используем метод .flow_from_directory.
# Создадим отдельные генераторы для тренировочных и тестовых изображений:

# Создаем обученную inception (подгружаем веса imagenet - потому обученная азаз)
inc_model=InceptionV3(include_top=False,
                      weights='imagenet',
                      input_shape=((3, 150, 150)))

#тут нужно указать какую аугментацию будем применять
bottleneck_datagen = ImageDataGenerator(rescale=1./255)  #генератор будет генерировать (ухты) аугментированные пикчи

#окей, просто подгружаем наши пикчи в генератор и генерируем
train_generator = bottleneck_datagen.flow_from_directory('data/img_train/',
                                        target_size=(150, 150),
                                        batch_size=32,
                                        class_mode=None,
                                        shuffle=False)

validation_generator = bottleneck_datagen.flow_from_directory('data/img_val/',
                                                               target_size=(150, 150),
                                                               batch_size=32,
                                                               class_mode=None,
                                                               shuffle=False)

print(' ')
print('-'*50)
print('''
  ___                  _   _       __   ______  ___ _
 |_ _|_ _  __ ___ _ __| |_(_)___ _ \ \ / /__ / | _ |_)_ _  __ _ _ _ _  _
  | || ' \/ _/ -_) '_ \  _| / _ \ ' \ V / |_ \ | _ \ | ' \/ _` | '_| || |
 |___|_||_\__\___| .__/\__|_\___/_||_\_/ |___/ |___/_|_||_\__,_|_|  \_, |
                 |_|                                                |__/
            ''')
print('Step_1')
print('Fetching the bottleneck features from pretrained InceptionV3 as numpy arrays')
print('-'*50)

if not os.path.exists('bottleneck_features/'):
    os.makedirs('bottleneck_features/')
    print('Directory "bottleneck_features/" has been created')
    print(' ')

# Прогоним аугментированные изображения через обученную Inception и сохраним вывод в виде numpy массивов
# - получим расширенный набор данных для обучения еще и помеченный как надо:
print('Saving bn_features_train.npy to bottleneck_features/')
bottleneck_features_train = inc_model.predict_generator(train_generator, 2000)
np.save(open('bottleneck_features/bn_features_train.npy', 'wb'), bottleneck_features_train)

print('Saving bn_features_validation.npy to bottleneck_features/')
bottleneck_features_validation = inc_model.predict_generator(validation_generator, 2000)
np.save(open('bottleneck_features/bn_features_validation.npy', 'wb'), bottleneck_features_validation)
print('Finished')


import numpy as np
import matplotlib.pyplot as plt
from control.matlab import *

m1 = 0.8
m2 = 0.2
k1 = 100

c1 = 1
c2 = 0.3
Ks = 100

M = np.matrix([[m1, 0],[0, m2]])
C= np.matrix([[c1+c2, -c2],[-c2, c2]])
F = np.matrix([[Ks],[0]])

iM = np.linalg.inv(M)
Bp = np.concatenate([np.zeros((2,1)), iM*F])
Cp = [0,1,0,0]
Dp = 0

plt.figure(1)
plt.subplot(2,1,1)
plt.ylim(-60, 40)
plt.yticks(np.arange(-60, 41, 20))
plt.xlim(1e0, 1e2)
plt.subplot(2,1,2)
plt.ylim(-360, 0)
plt.yticks(np.arange(-360, 1, 90))
plt.xlim(1e0, 1e2)
fig1, fig2 = plt.gcf().axes # subplotのハンドラを取得

delta = np.arange(-1, 1, 0.1)
for i in range(len(delta)):
    k2 = 300+300*0.2*delta[i]
    K = np.matrix([[k1+k2, -k2],[-k2, k2]])
    Ap = np.concatenate([np.concatenate([np.zeros((2,2,)), np.eye(2)], axis = 1), np.concatenate([-iM*K, -iM*C], axis = 1)])
    P = ss(Ap, Bp, Cp, Dp)
    mag, phase, om = bode(P, plot = False)
    plt.sca(fig1)
    plt.semilogx(om, 20*np.log10(mag))
    plt.sca(fig2)
    plt.semilogx(om, phase*180/np.pi)
plt.sca(fig1)
plt.xticks([])
plt.show()