from django.contrib.auth.hashers import CryptPasswordHasher
from django.db import models
from django.contrib.auth.models import User
from django.db.models.deletion import CASCADE, DO_NOTHING, PROTECT
from django.db.models.fields import CharField, IntegerField, related
from django.core.validators import MaxValueValidator, RegexValidator
from django.utils import tree
import random,calendar
from datetime import datetime
import rstr

from . import validators

format_port = {
    "type2": "Type 2",
    "ccs": "CCS",
    "type1": "Type 1",
    "chademo": "CHAdeMO",
    "tesla_ccs": "CCS",
    "tesla_suc": "Tesla Supercharger"
}

 # Customer
class Customer(models.Model):
    user = models.OneToOneField(User, related_name="customer",on_delete=models.CASCADE)
    has_expired_bills = models.BooleanField()

    def __str__(self):
        return f"Customer with ID:{self.id} and username {self.user.username} and {self.user.password}and has expired {self.has_expired_bills}"

    def update(self):

        booleans = [True, False]

        self.has_expired_bills = random.choice(booleans)
        self.save()

class APIKey(models.Model):
    customer = models.OneToOneField(User, related_name="apikey", on_delete = models.CASCADE)
    apikey = models.CharField(max_length=14)

    @classmethod
    def generate(cls,user):
        randomapikey = rstr.xeger(r'[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}')
        obj = cls.objects.create(
            customer = user,
            apikey = randomapikey
        ).save()
        return randomapikey

 # Individual Bill
class Bill(models.Model):
    customer = models.ForeignKey(User, related_name="bills", on_delete=models.CASCADE, null=True) # Many to One relationship with Customers
    date_created = models.DateTimeField() # Updates automatically the time the object is saved
    total = models.FloatField()
    is_paid = models.BooleanField()

    def __str__(self):
        return f"Bill with ID:{self.id} belongs to {self.customer.get_username()} and is_paid is {self.is_paid} created {self.date_created}."

    def payoff(self):

        self.is_paid = True
        self.save()

 # Monthly Bill expires every 1st of month
class MonthlyBill(models.Model):
    customer = models.ForeignKey(User, related_name="monthlybills", on_delete=models.CASCADE, null=True) # Many to One relationship with Customers
    monthly_total = models.FloatField(default=0)
    start_date = models.DateField()
    end_date = models.DateField(null=True, default=None)

    def __str__(self):
        return f"Monthly bill {self.id}, {self.start_date} to {self.end_date} belongs to {self.customer.username}."

    def get_current(user):
        m = MonthlyBill.objects.get_or_create(customer = user, end_date=None)
        return m.first()

    def payoff(self):

        self.monthly_total = -self.monthly_total
        self.save()

 # Stored cards for customer
class Card(models.Model):
    customer = models.ForeignKey(User, related_name="cards", on_delete=models.CASCADE) # Many to One relationship with Customers
    card_no = models.IntegerField()

    def __str__(self):
        first_dig = self.card_no / 10000
        last_dig = self.card_no % 10000
        return f"{first_dig}********{last_dig}"

 # To represent class Car

# Filled by reference
class Brands(models.Model):
    id = models.CharField(max_length=100 ,primary_key=True)
    name = models.CharField(max_length=100)

    def __str__(self):
        return f"Brand {self.name} has ID: {self.id}."

    @classmethod
    def create(cls, **kwargs):
        brand = cls.objects.create(
            id = kwargs['id'],
            name = kwargs['name']
        )
        return brand

class Ports(models.Model):
    id = models.IntegerField(primary_key=True)
    title = models.CharField(max_length=100)

    def __str__(self):
        return f"Port with ID: {self.id} and title {self.title}."

    @classmethod
    def create(cls, **kwargs):
        port = cls.objects.create(
            id=kwargs['ID'],
            title=kwargs['Title']
        )
        return port

# Filled by model car
class ACcharger(models.Model):
    id = models.AutoField(primary_key=True)
    usable_phases = models.IntegerField()
    ports = models.ManyToManyField(Ports, related_name='portas') # Many to Many relationship to existing Ports
    max_power = models.FloatField()
    # Might want to insert power_per_charging_point afterwards

    def __str__(self):
        return f"AC charger with {self.usable_phases} phases, available ports: {self.ports.all()} and {self.max_power} max power."

 # kwargs is data['data']['ac_charger']
    @classmethod
    def create(cls,**kwargs):
        charger = cls.objects.create(
            usable_phases = kwargs['usable_phases'],
            max_power = kwargs['max_power']
        )
        for i in kwargs['ports']:
            formatted_port = format_port[i]
            #print(formatted_port)
            port = Ports.objects.filter(title__startswith=formatted_port)
            for portinstance in port:
                #print(portinstance)
                charger.ports.add(portinstance)
        #print(charger.ports)
        return charger

class chargingCurve(models.Model):
    id = models.AutoField(primary_key=True)
    percentage = models.FloatField(validators=[validators.validate_percentage])
    power = models.FloatField()

    def __str__(self):
        return f"Charging percentage {self.percentage} with power {self.power}."

class DCcharger(models.Model):
    id = models.AutoField(primary_key=True)
    ports = models.ManyToManyField(Ports) # Many to Many relationship to existing Ports
    max_power = models.FloatField()
    charging_curve = models.ManyToManyField(chargingCurve)

    def __str__(self):
        return f"DC charger with available ports: {self.ports.all()} and {self.charging_curve.all()}."

# kwargs is data['data']['dc_charger']
    @classmethod
    def create(cls,**kwargs):
        dcharger= cls.objects.create(
            max_power = kwargs['max_power']
        )
        for i in kwargs['ports']:
            formatted_port = format_port[i]
            port = Ports.objects.filter(title__startswith=f"{formatted_port}")
            for portinstance in port:
                dcharger.ports.add(portinstance)

        for i in kwargs['charging_curve']:
            curve = chargingCurve.objects.get_or_create(
                percentage=i['percentage'],
                power=i['power']
            )
            dcharger.charging_curve.add(curve[0])

        return dcharger

class CarBase(models.Model):
    id = models.CharField(max_length=100, primary_key=True)
    type = models.CharField(max_length=4, null=True)
    brand = models.ForeignKey(Brands, on_delete=PROTECT, null=True)
    model = models.CharField(max_length=100)
    release_year = models.IntegerField(validators=[validators.validate_year],null=True)
    usable_battery_size = models.FloatField(validators=[validators.validate_percentage])
    ac_charger = models.OneToOneField(ACcharger, on_delete=PROTECT, null=True)
    dc_charger = models.OneToOneField(DCcharger, on_delete=PROTECT, null=True)
    average_consumption = models.FloatField(validators=[validators.validate_percentage])

    def __str__(self):
        return f"Car with ID: {self.id},model {self.model} and type {self.type}."

 # kwargs is data['data']
    @classmethod
    def create(cls,**kwargs):
        car = cls.objects.create(
            id = kwargs['id'],
            type = kwargs['type'],
            model = kwargs['model'],
            release_year = kwargs['release_year'],
            usable_battery_size = kwargs['usable_battery_size'],
            average_consumption = kwargs['energy_consumption']['average_consumption']
        )
        brand_id = kwargs['brand_id']
        brand_to_enter = Brands.objects.get(id=brand_id)
        car.brand = brand_to_enter
        ac_charger = ACcharger.create(**kwargs['ac_charger'])
        #print(ac_charger)
        car.ac_charger=ac_charger
        if kwargs['dc_charger'] != None:
            dc_charger = DCcharger.create(**kwargs['dc_charger'])
            car.dc_charger = dc_charger

        return car

class Car(models.Model):
    id = models.AutoField(primary_key=True)
    car = models.ForeignKey(CarBase, related_name="carbase",on_delete=models.DO_NOTHING, null=True)
    customer = models.ForeignKey(User, related_name="cars", on_delete=models.CASCADE, null=True)

    def __str__(self):
        return f"Car: {self.id} belongs to {self.customer.id}."

    @classmethod
    def create(cls, **kwargs):
        # Perhaps more cars in each user
        for i in User.objects.all():
            cars = CarBase.objects.all()
            random_car = random.choice(cars)
            newCar = cls.objects.create(
                car = random_car,
                customer = i
            )
            newCar.save()
        return newCar

# To reprsent class Station
class Operator(models.Model):
    id = models.IntegerField(primary_key=True)
    website_url = models.URLField(null=True, blank=True)
    contact_email = models.EmailField(null=True, blank=True)
    title = models.CharField(max_length=100, null=True, blank=True)

    def __str__(self):
        return f"Operator {self.title}, website {self.website_url}, email {self.contact_email}."

# Filled by reference
class UsageType(models.Model):
    id = models.IntegerField(primary_key=True)
    Title = models.CharField(max_length=100)
    IsPayAtLocation = models.BooleanField(null=True, blank=True)
    IsMembershipRequired = models.BooleanField(null=True, blank=True)
    IsAccessKeyRequired = models.BooleanField(null=True, blank=True)

    def __str__(self):
        return f"Is {self.Title} with ID: {self.id}, IsPayAtLocation:{self.IsPayAtLocation}, IsMembershipRequired:{self.IsMembershipRequired}, IsAccessKeyRequired:{self.IsAccessKeyRequired}.\n"

 # kwargs is data['UsageTypes']
    @classmethod
    def create(cls,**kwargs):
        usagetype = cls.objects.create(
            id = kwargs['ID'],
            Title = kwargs['Title'],
            IsPayAtLocation = kwargs['IsPayAtLocation'],
            IsMembershipRequired = kwargs['IsMembershipRequired'],
            IsAccessKeyRequired = kwargs['IsAccessKeyRequired']
        )
        return usagetype

 # Filled by reference
class StatusType(models.Model):
    id = models.IntegerField(primary_key=True)
    Title = models.CharField(max_length=100)
    IsOperational = models.BooleanField(null=True, blank=True)

    def __str__(self):
        return f"Is {self.Title}"

 # kwargs is data['StatusTypes']
    @classmethod
    def create(cls,**kwargs):
        status = cls.objects.create(
            id = kwargs['ID'],
            Title = kwargs['Title'],
            IsOperational = kwargs['IsOperational']
        )
        return status

 # Filled by station.create
class AddressInfo(models.Model):
    id = models.IntegerField(primary_key=True)
    title = models.CharField(max_length=100)
    addressLine = models.CharField(max_length=100)
    town = models.CharField(max_length=100, null=True)
    stateOrProvince = models.CharField(max_length=100, null=True)
    postCode = models.CharField(max_length=10,null=True)
    countryId = models.IntegerField()
    latitude = models.DecimalField(max_digits=9, decimal_places=6)
    longtitude = models.DecimalField(max_digits=9, decimal_places=6)
    contact_telephone = models.CharField(max_length=13, null=True, blank=True)
    access_comments = models.CharField(max_length=1000, null=True, blank=True)

    def __str__(self):
        return f"{self.addressLine}, {self.town}, {self.postCode}."

 # kwargs is data['AddressInfo']
    @classmethod
    def create(cls,**kwargs):
        check = AddressInfo.objects.filter(id = kwargs['ID'])
        if check:
            return None
        address = cls.objects.create(
            id = kwargs['ID'],
            title = kwargs['Title'],
            addressLine = kwargs['AddressLine1'],
            town = kwargs['Town'],
            stateOrProvince = kwargs['StateOrProvince'],
            postCode = kwargs['Postcode'],
            countryId = kwargs['CountryID'],
            latitude = kwargs['Latitude'],
            longtitude = kwargs['Longitude'],
            contact_telephone = kwargs['ContactTelephone1'],
            access_comments = kwargs['AccessComments']
        )
        return address

 # Filled by reference
class CurrentType(models.Model):
    id = models.IntegerField(primary_key=True)
    title = models.CharField(max_length=50)

    def __str__(self):
        return f"Current type of {self.title} and id {self.id}."

 # kwargs is data['CurrentTypes']
    @classmethod
    def create(cls,**kwargs):
        currType = cls.objects.create(
            id = kwargs['ID'],
            title = kwargs['Title']
        )
        return currType

 # Filled by stations.create

class Provider(models.Model):
    id = models.AutoField(primary_key=True)
    costPerkWh = models.FloatField()
    name = models.CharField(max_length=10)

    def __str__(self):
        return f"{self.name} provider with cost per kWh {self.costPerkWh}"

    @classmethod
    def create(cls, **kwargs):
        provider = cls.objects.create(
            costPerkWh = kwargs['costPerkWh'],
            name = kwargs['name']
        )
        provider.save()
        return provider

class Station(models.Model):
    id = models.IntegerField(primary_key=True)
    providers = models.ManyToManyField(Provider, related_name="providers")
    operators = models.ManyToManyField(Operator, related_name="operates")
    usageType = models.ManyToManyField(UsageType, blank=True)
    statusType = models.ManyToManyField(StatusType, blank=True)
    addressInfo = models.OneToOneField(AddressInfo,related_name="belongs_to",on_delete=models.CASCADE, null=True)
    usageCost = models.CharField(null=True, blank=True, max_length=100)
    #userComments in UserComments table, accessed by station.UserComments
    generalComments = models.CharField(max_length=1000, null=True, blank=True)

    def __str__(self):
        return f"Station: {self.id}, Provider:{self.providers.all().first().name}"

# kwargs is data
    @classmethod
    def create(cls,**kwargs):
        addressI = AddressInfo.create(**kwargs['AddressInfo'])
        if addressI != None:
                addressI.save()

        check = Station.objects.filter(id = kwargs['ID'])
        if check:
            return None

        station = cls.objects.create(
            id = kwargs['ID'],
            generalComments = kwargs['GeneralComments'],
            usageCost = kwargs['UsageCost'],
            addressInfo= addressI
            )

        station.providers.add(random.choice(Provider.objects.all()))

        opInfo = kwargs['OperatorInfo']
        if opInfo != None:
            operator = Operator.objects.get_or_create(
                id = opInfo['ID'],
                website_url = opInfo['WebsiteURL'],
                contact_email = opInfo['ContactEmail'],
                title = opInfo['Title']
            )
            if operator[1]:
                operator[0].save
            station.operators.add(operator[0])

        connectionsInfo = kwargs['Connections']
        for i in connectionsInfo:
            point = Point.objects.create(
                id = i['ID'],
                voltage = i['Voltage'],
                powerKW = i['PowerKW'],
                quantity = i['Quantity']
            )
            if i['Level'] is not None:
                point.protocol = i['Level']['Title']
            #print(point)

            port = Ports.objects.get(id=i['ConnectionTypeID'])
            #print(port)
            point.ports.add(port)
            if i['CurrentTypeID'] != None:
                currType = CurrentType.objects.get(id=i['CurrentTypeID'])
                point.current_type.add(currType)

            if i['StatusTypeID'] != None:
                status = StatusType.objects.get(id=i['StatusTypeID'])
                point.status_type.add(status)

            point.station = station
            point.save()
            #print(connection)

        if kwargs['UsageTypeID'] != None:
            usagetype = UsageType.objects.get(id=kwargs['UsageTypeID'])
            station.usageType.add(usagetype)

        if kwargs['StatusTypeID'] != None:
            statustype = StatusType.objects.get(id=kwargs['StatusTypeID'])
            station.statusType.add(statustype)

        if kwargs['UserComments'] != None:
            for i in kwargs['UserComments']:
                ucomm = UserComments.objects.create(
                    station = station,
                    username = i['UserName'],
                    comment = i['Comment'],
                    rating = i['Rating'],
                )
                if i['CheckinStatusTypeID'] != None:
                    ucomm.checkinStatus.add(CheckinStatus.objects.get(id=i['CheckinStatusTypeID']))

        if kwargs['MediaItems'] != None:
            for i in kwargs['MediaItems']:
                m = station.mediaItems.get_or_create(
                    id = i['ID'],
                    itemUrl = i['ItemURL'],
                )
                if m[1]:
                    m[0].save()

        return station

    @classmethod
    def update(cls, **kwargs):
        i = kwargs["ID"]
        j = kwargs["AddressInfo"]["ID"]

        station = Station.objects.get(id=int(i))
        address = AddressInfo.objects.get(id=int(j))
        station.addressInfo=address
        station.save()

    @property
    def getID(self):
        return self.id

    @property
    def rating(self):
        count=0
        rating=0
        allComms = UserComments.objects.select_related('station').filter(station__id=self.id)

        for i in allComms:
            if i.rating != None:
                rating += i.rating
                count += 1
        if count==0:
            return (0,0)
        return (rating/count, count)

class Point(models.Model):
    id = models.IntegerField(primary_key=True)
    current_type = models.ManyToManyField(CurrentType, related_name="exists_at",blank=True)
    voltage = models.IntegerField(null=True, blank=True)
    powerKW = models.FloatField(null=True)
    quantity = models.IntegerField(null=True)
    status_type = models.ManyToManyField(StatusType, blank=True) #One status type to many connections
    station = models.ForeignKey(Station, related_name="comments", on_delete=models.DO_NOTHING, null=True)
    ports = models.ManyToManyField(Ports, related_name="exist_at")
    protocol = models.CharField(max_length=20, default="Level 2 : Medium (Over 2kW)")

    def __str__(self):
        return f"Point: {self.id}, Ports: {self.ports.all()}"

 # Filled by stations.create
class MediaTypes(models.Model):
    id = models.IntegerField(primary_key=True)
    itemUrl = models.URLField(null=True)
    station = models.ForeignKey(Station, related_name="mediaItems", on_delete=models.CASCADE)

    def __str__(self):
        return f"Media ID: {self.id} and URL: {self.itemUrl} and station {self.station}."

 # Filled by reference
class CheckinStatus(models.Model):
    id = models.IntegerField(primary_key=True)
    title = models.CharField(max_length=100)

    def __str__(self):
        return f"Checkin status {self.title} with ID: {self.id}."

# kwargs is data['CheckinStatusTypes']
    @classmethod
    def create(cls, **kwargs):
        checkin = cls.objects.create(
            id = kwargs['ID'],
            title = kwargs['Title']
        )
        return checkin

class UserComments(models.Model):
    id = models.AutoField(primary_key=True)
    station = models.ForeignKey(Station, related_name="UserComments", on_delete=models.CASCADE)
    username = models.CharField(max_length=100)
    comment = models.CharField(max_length=1000, blank=True, null=True)
    rating = models.IntegerField(null=True, blank=True)
    customer = models.ForeignKey(User, related_name="myComments",on_delete=models.CASCADE, null=True, blank=True)
    checkinStatus = models.ManyToManyField(CheckinStatus)

class Session(models.Model):
    CASH = 'Cash'
    DEBIT = 'Debit Card'
    CREDIT = 'Credit'
    PAYMENT_OPTIONS = [
        (CASH, 'PayinCash'),
        (DEBIT, 'PaywithCard'),
        (CREDIT, 'Payattheendofmonth')
    ]

    id = models.AutoField(primary_key=True)
    provider = models.ForeignKey(Provider, related_name="hasmade", on_delete=models.DO_NOTHING, null=True)
    customer = models.ForeignKey(User, related_name="sessions", on_delete=models.CASCADE)
    vehicle = models.ForeignKey(Car, related_name="vehicle", on_delete=models.DO_NOTHING, null=True)
    station = models.ForeignKey(Station, related_name="sessions", on_delete=models.DO_NOTHING, null=True)
    point = models.ForeignKey(Point, related_name="points", on_delete=models.DO_NOTHING, null=True)
    connectionTime = models.DateTimeField(null=True)
    disconnectTime = models.DateTimeField(null=True)
    doneChargingTime = models.DateTimeField(null=True)
    kWhDelivered = models.FloatField(null=True)
    payment = models.CharField(max_length=10, choices=PAYMENT_OPTIONS, default=CASH,)

    def __str__(self):
        return f"Session with ID {self.id} Charged with {self.provider.name} , transfered totally {self.kWhDelivered} KWh, created {self.connectionTime}."

    @classmethod
    def create(cls, **kwargs):
        users = User.objects.all()
        random_user = random.choice(users)

        users_car = random_user.cars.all()
        random_vehicle = random.choice(users_car)

        stations = Station.objects.all()
        random_station = random.choice(stations)

        random_provider = random.choice(random_station.providers.all())
        random_point = random.choice(random_station.comments.all())

        random_paymentOpt = random.choice(['Credit','Debit Card','Cash'])

        session = cls.objects.create(
            customer = random_user,
            vehicle = random_vehicle,
            provider = random_provider,
            station = random_station,
            point = random_point,
            payment = random_paymentOpt,
            connectionTime = kwargs['connectionTime'],
            disconnectTime = kwargs['disconnectTime'],
            doneChargingTime = kwargs['doneChargingTime'],
            kWhDelivered = kwargs['kWhDelivered']
        )
        userinputs = []
        if kwargs['userInputs'] is not None:
            for i in kwargs['userInputs']:
                user_inputs = UserInput.create(**i)
                user_inputs.customer = random_user
                userinputs.append(user_inputs)
                user_inputs.save()

        session.userInputs.set(userinputs)

        if random_paymentOpt == 'Credit':
            is_paid=False
        else:
            is_paid=True
            b = Bill.objects.create(customer = random_user,
                                    date_created = session.connectionTime,
                                    total = session.price,
                                    is_paid = is_paid)
        if is_paid==False:
            time = session.connectionTime
            date = datetime.strptime(time,"%Y-%m-%d %H:%M:%S.00+00:00")
            lastday = calendar.monthrange(date.year,date.month)[1]
            start_date = datetime(date.year, date.month, 1).strftime("%Y-%m-%d")
            end_date = datetime(date.year, date.month, lastday).strftime("%Y-%m-%d")
            customer = random_user.customer
            customer.has_expired_bills = True
            customer.save()
            m = MonthlyBill.objects.get_or_create(start_date = start_date, end_date = end_date , customer = random_user)
            if m[1]:
                m[0].save()
            if m[0].monthly_total < 0:
                m[0].monthly_total = session.price
            else:
                m[0].monthly_total = m[0].monthly_total+session.price
            m[0].save()

        return session

    @property
    def price(self):
        return (self.kWhDelivered)*(self.provider.costPerkWh)

class UserInput(models.Model):
    session = models.ForeignKey(Session, related_name="userInputs", on_delete=models.CASCADE, null=True)
    customer = models.ForeignKey(User, related_name="inputs", on_delete=models.CASCADE, null=True)
    WhPerMile = models.IntegerField(null=True)
    kWhRequested = models.FloatField(null=True)
    milesRequested = models.IntegerField(null=True)
    minutesAvailable = models.IntegerField(null=True)
    modifiedAt = models.DateTimeField(null=True)
    requestedDeparture = models.DateTimeField(null=True)

    def __str__(self):
        return f"User {self.customer.username} requested {self.kWhRequested} kWh, {self.milesRequested} miles for a car with consumption {self.WhPerMile} Wh/mile. He requested departure at {self.requestedDeparture} minutes from {self.modifiedAt} and has {self.minutesAvailable}."

    @classmethod
    def create(cls, **kwargs):
        userInput = cls.objects.create(
            WhPerMile = kwargs['WhPerMile'],
            kWhRequested = kwargs['kWhRequested'],
            milesRequested = kwargs['milesRequested'],
            minutesAvailable = kwargs['minutesAvailable'],
            modifiedAt = kwargs['modifiedAt'],
            requestedDeparture = kwargs['requestedDeparture']
        )

        return userInput


import math
import numpy as np
from scipy.misc import derivative
from scipy.interpolate import CubicSpline
import matplotlib.pyplot as plt
from sympy import *

'''
y = ((1+x^8)^(3/2))/(12x^12)
'''

'''
 sqrt(x-1)*(3x+2)/(4x^2)
'''

polinom = lambda x: (pow((1 + pow(x, 8)), (3/2)) / (12 * pow(x, 12)))

def Show_6th_deriv():
    x = Symbol('x')
    y = (pow((1 + pow(x, 8)), (3/2)) / (12 * pow(x, 12)))
    der_6_y = (((((y.diff(x)).diff(x)).diff(x)).diff(x)).diff(x)).diff(x)
    print('Исходная функция: ', y)
    print('Производная 6-го порядка: ', der_6_y)

def Get_sixth_deriv(polinom, x):
    return derivative(polinom, x, n = 6, order = 9, dx=1e-5)

def Get_1st_deriv(polinom, x):
    return derivative(polinom, x, dx=1e-5)

def Get_2st_deriv(polinom, x):
    return derivative(polinom,x, n=2, order=5, dx=1e-5)

def product(x, list_x, n):
    mul = 1
    for i in range(len(list_x)):
        if i:
            mul *= x - list_x[i - 1]
        yield mul

def Newthon_forward(list_x, list_y, x):
    C = []

    forward_newthon_polinom = 0

    for n in range(len(list_x)):
        p = product(list_x[n], list_x, n + 1)
        C.append((list_y[n] - sum(C[k] * next(p) for k in range(n))) / next(p))

    forward_newthon_polinom = sum(C[k] * p for k, p in enumerate(product(x,list_x, len(C))))

    return forward_newthon_polinom

def Newthon_reverse(list_x, list_y, x):
    C = []

    reverse_newthon_polinom = 0

    for n in range(len(list_x)):
        p = product(sorted(list_x, reverse=True)[n], sorted(list_x, reverse=True), n + 1)
        C.append((sorted(list_y, reverse=True)[n] - sum(C[k] * next(p) for k in range(n))) / next(p))

    reverse_newthon_polinom = sum(C[k] * p for k, p in enumerate(product(x, sorted(list_x, reverse=True), len(C))))

    return reverse_newthon_polinom

def Lagrange_forward(list_x, list_y, x):

    forward_lagrange_polinom = 0
    for j in range(len(list_y)):
        product = 1 #числитель
        product1 = 1 # знаменатель
        for i in range(len(list_x)):
            if i==j:
                product *= 1
                product1 *= 1
            else:
                product *= (x - list_x[i])
                product1 *= (list_x[j] - list_x[i])
        forward_lagrange_polinom += list_y[j] * (product / product1)
    return forward_lagrange_polinom

def Lagrange_reverse(list_x, list_y, x):

    reverse_lagrange_polinom = 0
    for j in range(len(list_y) - 1, -1, -1):
        product = 1 #числитель
        product1 = 1 # знаменатель
        for i in range(len(list_x) - 1, -1, -1):
            if i==j:
                product *= 1
                product1 *= 1
            else:
                product *= (x - list_x[i])
                product1 *= (list_x[j] - list_x[i])
        reverse_lagrange_polinom += list_y[j] * (product / product1)
    return reverse_lagrange_polinom

def Cubic_Spline(x_list, y_list, x_dop_l, x_dop_r, x):
    N = len(x_list)
    matrix = [[0 for j in range(3 * N)] for i in range(3 * N)]
    rightVector = [0 for i in range(3 * N)]
    for line in range(N - 1):
        matrix[line][line + 1] = 1
        matrix[line][N + line + 1] = -(x_list[line + 1] - x_list[line]) / 2
        matrix[line][2 * N + line + 1] = ((x_list[line + 1] - x_list[line]) ** 2) / 6
        rightVector[line] = (y_list[line + 1] - y_list[line]) / (x_list[line + 1] - x_list[line])
        matrix[N - 1 + line][line + 1] = 1
        matrix[N - 1 + line][line] = -1
        matrix[N - 1 + line][N + line + 1] = -(x_list[line + 1] - x_list[line])
        matrix[N - 1 + line][2 * N + line + 1] = ((x_list[line + 1] - x_list[line]) ** 2) / 2
        matrix[N * 2 - 2 + line][N + line + 1] = 1
        matrix[N * 2 - 2 + line][N + line] = -1
        matrix[N * 2 - 2 + line][2 * N + line + 1] = -(x_list[line + 1] - x_list[line + 1])
    matrix[3 * N - 3][0] = x_list[1] - x_list[0]
    matrix[3 * N - 3][N] = ((x_list[1] - x_list[0]) ** 2) / 2
    matrix[3 * N - 3][2 * N] = -((x_list[1] - x_list[0]) ** 2) / 6
    rightVector[3 * N - 3] = y_list[1] - y_list[0]
    matrix[3 * N - 2][2 * N - 1] = 1
    rightVector[3 * N - 2] = x_dop_r
    matrix[3 * N - 1][N] = 1
    matrix[3 * N - 1][2 * N] = -(x_list[1] - x_list[0])
    rightVector[3 * N - 1] = x_dop_l

    matrix_coef = np.linalg.solve(matrix, rightVector)

    n = len(x_list)
    i = 1
    while not (x_list[i - 1] <= x and x <= x_list[i]): i += 1
    return y_list[i] + matrix_coef[i] * (x - x_list[i]) + matrix_coef[n + i] * (((x - x_list[i])) ** 2) / 2 + matrix_coef[2 * n + i] * ((x - x_list[i]) ** 3) / 6

def Accuracy(x, model_res):
    return math.fabs(polinom(x) - model_res)

def Plot_func_and_6th_deriv():
    x = np.arange(-10, 10, 0.5)
    y = pow((1 + pow(x, 8)), (3/2)) / (12 * pow(x, 12))
    sixth_der = 107520.0*x**30*(x**8 + 1)**(-4.5) - 126720.0*x**22*(x**8 + 1)**(-3.5) + 66240.0*x**14*(x**8 + 1)**(-2.5) + 42000.0*x**6*(x**8 + 1)**(-1.5) + 274680.0*(x**8 + 1)**(-0.5)/x**2 - 1106280.0*(x**8 + 1)**0.5/x**10 + 742560*(x**8 + 1)**1.5/x**18

    plt.subplot(221)
    plt.plot(x, y, 'r')
    plt.title('Function')
    plt.grid(True)
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(222)
    plt.plot(x, sixth_der, 'b')
    plt.title('Sixth derivative')
    plt.grid(True)
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.show()

def Plot_Lagrange_results():
    x = [i for i in np.linspace(1, 20, 6)]
    y = [polinom(i) for i in x]
    NewX = [i for i in np.linspace(np.min(x), np.max(x), 100)]
    NewY_forw = [Lagrange_forward(x, y, i) for i in NewX]
    NewY_rev = [Lagrange_reverse(x, y, i) for i in NewX]
    Y = [polinom(i) for i in NewX]#[np.interp(i, x, y) for i in NewX]

    plt.subplot(221)
    plt.plot(x, y, 'o', NewX, NewY_forw)
    plt.title("Forward Lagrange")
    plt.legend(['Узловые значения',"Forward Lagrange"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(222)
    plt.plot(x, y, 'o', NewX, NewY_rev)
    plt.title("Reverse Lagrange")
    plt.legend(['Узловые значения',"Reverse Lagrange"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(223)
    plt.plot(x, y, 'o', NewX, Y)
    #plt.title("График исходной функции на этом отрезке")
    plt.legend(['Узловые значения', "Исходная функция"])
    plt.xlabel("X")
    plt.ylabel("Y")
    plt.savefig('Метод Лагранжа.png')
    plt.show()

def Plot_Newthon_results():
    x = [i for i in np.linspace(1, 20, 6)]
    y = [polinom(i) for i in x]
    NewX = [i for i in np.linspace(np.min(x), np.max(x), 100)]
    NewY_forw = [Newthon_forward(x, y, i) for i in NewX]
    NewY_rev = [Newthon_reverse(x, y, i) for i in NewX]
    Y = [polinom(i) for i in NewX]

    plt.subplot(221)
    plt.plot(x, y, 'o', NewX, NewY_forw)
    plt.title("Forward Newthon")
    plt.legend(['Узловые значения', "Forward Newthon"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(222)
    plt.plot(x, y, 'o', NewX, list(reversed(NewY_rev)))
    plt.title("Reverse Newthon")
    plt.legend(['Узловые значения', "Reverse Newthon"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(223)
    plt.plot(x, y, 'o', NewX, Y)
    # plt.title("График исходной функции на этом отрезке")
    plt.legend(['Узловые значения', "Исходная функция"])
    plt.xlabel("X")
    plt.ylabel("Y")
    plt.savefig('Метод Ньютона.png')
    plt.show()

def Plot_CubicSpline_results():
    x = [i for i in np.linspace(1, 20, 6)]
    y = [polinom(i) for i in x]
    NewX = [i for i in np.linspace(np.min(x), np.max(x), 100)]
    cs = CubicSpline(x, y)
    NewY = [cs(i) for i in NewX]
    Y = [Cubic_Spline(x, y, 0, 0, i) for i in NewX]
    Y_ = [polinom(i) for i in NewX]

    plt.subplot(221)
    plt.plot(x, y, 'o', NewX, NewY)
    plt.title("Numpy Cubic_Spline")
    plt.legend(['Узловые значения', "Numpy Cubic_spline"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(222)
    plt.plot(x, y, 'o', NewX, Y)
    plt.title("CubicSpline")
    plt.legend(['Узловые значения', "Cubic_spline"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.subplot(223)
    plt.plot(x, y, 'o', NewX, Y_)
    # plt.title("График исходной функции на этом отрезке")
    plt.legend(['Узловые значения', "Исходная функция"])
    plt.xlabel("X")
    plt.ylabel("Y")

    plt.savefig("Кубический сплайн.png")
    plt.show()

def Mod(x, list_x):
    mul = 1
    for i in range(len(list_x)):
        mul *= (x - list_x[i])

    return mul

def Plot_accuracy_graphs():
    x = [i for i in np.linspace(1, 20, 6)]
    y = [polinom(i) for i in x]
    x_test = [i for i in np.linspace(1, 20, 100)]
    cs = CubicSpline(x, y)
    Y_lagr_forw = np.array([Accuracy(i, Lagrange_forward(x, y, i)) for i in x_test])
    Y_lagr_rev = np.array([Accuracy(i, Lagrange_reverse(x, y, i)) for i in x_test])
    Y_newth_forw = np.array([Accuracy(i, Newthon_forward(x, y, i)) for i in x_test])
    Y_newth_rev = np.array([Accuracy(i, Newthon_reverse(x, y, i)) for i in x_test])
    #Y_ = np.array([Accuracy(i, np.interp(i, x, y)) for i in x_test])
    Y_spline = np.array([Accuracy(i, Cubic_Spline(x, y, Get_2st_deriv(polinom, 1), Get_2st_deriv(polinom, 20), i)) for i in x_test])

    modul_6th_der = [math.fabs(Get_sixth_deriv(polinom, i)) for i in x]

    Y_majoarnt = list(map(lambda z: math.fabs(z) / math.factorial(len(x) + 1), [max(modul_6th_der) * Mod(i, x) for i in x_test]))

    plt.title("Графики ошибок")
    plt.plot(x_test, Y_lagr_forw, 'g', x_test, Y_newth_forw, 'r', x_test, Y_newth_forw, 'c',x_test, Y_spline, 'm')#, x_test, Y_majoarnt, 'b')
    plt.legend(["Lagrange_forward", 'Newthon forward', 'Newthon reverse', 'Cubic Spline'])#, 'Мажоранта помилки'])
    plt.savefig('Accuracy.png')
    plt.show()

def Plot_Majorant(list_x):
    x_ = [i for i in np.linspace(1, 20, 6)]
    modul_6th_der = [math.fabs(Get_sixth_deriv(polinom, i)) for i in x_]

    Y = list(map(lambda x: math.fabs(x) / math.factorial(len(x_) + 1), [max(modul_6th_der) * Mod(i, x_) for i in list_x]))
    #print(Y)

    plt.plot(list_x, Y, 'r')
    plt.legend(['Мажоранта ошибки'])
    plt.savefig('Мажоранта ошибки')
    plt.show()

def ShowFunc():

    x = [i for i in np.linspace(1, 20, 6)]
    y = [polinom(i) for i in x]
    print('Forward Lagrange \n', Lagrange_forward(x, y, 5))
    print('Reverse Lagrange \n', Lagrange_reverse(x, y, 5))
    print("Forward Newthon \n", Newthon_forward(x, y, 5))
    print('Reverse Newthon \n', Newthon_reverse(x, y, 5))
    print("Cubic Spline \n", Cubic_Spline(x, y, 0, 0, 5))
    print('Проверка интерполяцией полиномом: ', np.interp(5, x, y))
    cs = CubicSpline(x, y)# библиотечный сплайн 3-й степени
    print('Проверка интерполяцией кубическим сплайном: ', cs(5))

    #выводим значения ошибки на "учебном" отрезке для каждого из методов из шагом в 5 раз меньше чем шаг интерполяции
    x_test = [i for i in np.linspace(1, 20, 50)]
    f = open('Results.txt', 'w')
    f.write('Lagrange: \n')
    f.write(str(np.mean([Accuracy(i, Lagrange_forward(x, y, i)) for i in x_test])))
    f.write('\n')
    f.write('Newthon Forward: \n')
    f.write(str(np.mean([Accuracy(i, Newthon_forward(x, y, i)) for i in x_test])))
    f.write('\n')
    f.write('Newthon reversed: \n')
    f.write(str(np.mean([Accuracy(i, Newthon_reverse(x, y, i)) for i in x_test])))
    f.write('\n')
    f.write('Cubic_Spline: \n')
    f.write(str(np.mean([Accuracy(i, Cubic_Spline(x, y, 0, 0, i)) for i in x_test])))
    f.write('\n')
    f.close()
    print([Accuracy(i, Lagrange_forward(x, y, i)) for i in x_test])
    print([Accuracy(i, Lagrange_reverse(x, y, i)) for i in x_test])
    print([Accuracy(i, Newthon_forward(x, y, i)) for i in x_test])
    print([Accuracy(i, Newthon_reverse(x, y, i)) for i in x_test])
    print([Accuracy(i, Cubic_Spline(x, y, 0, 0, i)) for i in x_test])
    #print([Accuracy(i, cs(i)) for i in x_test])
    #print([Accuracy(i, np.interp(i, x, y)) for i in x_test])

def Plot_together():
    x = np.linspace(1, 20, 1000)
    x_ = np.linspace(1, 20, 6)
    y_ = [polinom(i) for i in x_]
    y = pow((1 + pow(x, 8)), (3/2)) / (12 * pow(x, 12))
    y_newthon = [Newthon_forward(x_, y_, i) for i in x]
    y_lagrange = [Lagrange_forward(x_, y_, i) for i in x]
    y_newthon_reversed = [Newthon_reverse(x_, y_, i) for i in x]
    y_spline = [Cubic_Spline(x_, y_, 0, 0, i) for i in x]

    plt.plot(x_, y_, 'o', x, y,'r', x, y_spline, 'b', x, y_newthon, 'y', x, list(reversed(y_newthon_reversed)), 'c', x, y_lagrange, 'g')
    plt.legend(["Узловые значения", "Исходная функция","Сплайн", "Ньютон", "Обратный Ньютон", "Лагранж"])
    plt.savefig('Результаты.png')
    plt.show()

Plot_together()

Plot_Lagrange_results()
Plot_Newthon_results()
Plot_CubicSpline_results()
Plot_func_and_6th_deriv()

Plot_accuracy_graphs()

x_ = [i for i in np.linspace(1, 20, 100)]
Plot_Majorant(x_)
#ShowFunc()
print('\n')
#Show_6th_deriv()


import json

from django.http import HttpResponse
from django.shortcuts import render
from django.core.files.storage import FileSystemStorage
import random, os
from CnnVisualizer.core.vis import Visualizer
from cnn_visualizer.settings import BASE_DIR
from django.views.decorators.csrf import csrf_exempt
import cv2

def index_template(request):
    return render(request, 'visualizer_server/index.html')

@csrf_exempt
def upload_request(request):
    model_id = "nomodel"
    fs = ''
    if request.method == 'POST':
        uploaded_model = request.FILES['model']
        uploaded_ptn = request.FILES['pth']
        uploaded_image = request.FILES['image']
        fs = FileSystemStorage()
        model_id = "model" + str(random.randrange(10000))
        fs.save(model_id +'/' + uploaded_model.name, uploaded_model)
        fs.save(model_id +'/' + uploaded_ptn.name, uploaded_ptn)
        fs.save(model_id +'/' + uploaded_image.name, uploaded_image)
    response = HttpResponse(model_id)
    run_model(model_id)
    return response

def run_model(model_id):
    directory = os.path.join(BASE_DIR, 'media', model_id)
    net_source = ""
    net_ckpt = ""
    input_path =""
    for file in os.listdir(directory):
        if os.path.splitext(file)[1] == '.py':
            net_source = os.path.join(directory, file)
        elif os.path.splitext(file)[1] == '.pth':
            net_ckpt = os.path.join(directory, file)
        else:
            input_path = os.path.join(directory, file)

    # print(net_ckpt)
    # print(net_source)
    # print(input_path)
    os.mkdir(os.path.join(directory, 'output'))
    a = Visualizer(net_source,net_ckpt,input_path,os.path.join(directory, 'output'))
    #os.chdir(os.path.join(directory, 'output')) # change directory to process
    return_files,model_info = a.vis()

    # find the path to client
    parent = os.path.abspath(os.path.join(BASE_DIR, os.pardir))

    target_dir = os.path.join(parent,'client','public',model_id)
    if os.path.exists(target_dir):
        os.rmdir(target_dir)
    os.makedirs(target_dir,exist_ok=True)

    for file in return_files:
        output_path = os.path.join(target_dir,file['image_name'])
        cv2.imwrite(output_path,file['image'])
        cv2.imwrite(os.path.join(directory,'output',file['image_name']), file['image'])
    with open(os.path.join(target_dir,'output.json'),'w') as f:
        json.dump(model_info,f)
    with open(os.path.join(directory,'output','output.json'),'w') as f:
        json.dump(model_info,f)

def get_images(request):
    return HttpResponse(request)

def get_model(request):
    return HttpResponse(request)

def auth(request):
    response = HttpResponse(request)
    response['Access-Control-Allow-Credentials'] = True
    return response


import asyncio
from pathlib import Path
from typing import Dict, Any

import aiohttp
import pandas as pd
from loguru import logger

class RateLimitedException(Exception):
    """Rate api limited exception"""

def load_data_from(filename: str, path: str) -> pd.DataFrame:
    base_path = Path(__file__).resolve().parent
    directory = base_path / path
    current_path = directory / filename
    data_file = pd.read_csv(current_path)
    return data_file

def clean_data(data: pd.DataFrame) -> pd.DataFrame:
    df_copy = data.copy()
    df_copy[["Participante", "IP"]] = df_copy["Nombre;GDPR IP"].str.split(
        ";", expand=True
    )
    return df_copy

async def fetch_location_participan(ip: str) -> Dict[str, Any]:
    async with aiohttp.ClientSession() as session:
        async with session.get(f"http://ip-api.com/json/{ip}") as response:
            logger.debug(f"Status: {response.status}")
            if response.status == 429:
                raise RateLimitedException(f"Could not fetch location this ip: {ip}")

            location = await response.json()
            return location

async def add_location_to_each(records_session: list[Dict]) -> pd.DataFrame:
    waiting_time_request = 20  # seconds
    new_dataframe = pd.DataFrame(columns=["Nombre", "IP", "Pais", "Ciudad", "Region"])
    count = 0
    total_len = len(records_session)
    while len(records_session) > 0:
        ip = records_session[0]["IP"]
        name = records_session[0]["Participante"]
        try:
            location = await fetch_location_participan(ip)
            logger.info(f"Name: {name}, Index:{count} - {location}")
            new_row = {
                "Nombre": name,
                "IP": ip,
                "Pais": location["country"],
                "Ciudad": location["city"],
                "Region": location["regionName"]
            }
            new_dataframe.loc[len(new_dataframe)] = new_row
            count += 1
            records_session.pop(0)  # remove first item
        except RateLimitedException as rt:
            logger.error(rt)
            await asyncio.sleep(waiting_time_request)

        logger.debug(f"Initial len: {total_len}, Current len: {count},  Len DataFrame: {new_dataframe.shape}")
    return new_dataframe

async def main():
    data_virtual_session = load_data_from("Listado_de_participantes_reto_1.csv", 'data')
    data_virtual_session = clean_data(data_virtual_session)
    records_session = data_virtual_session.to_dict(orient="records")
    new_dataframe = await add_location_to_each(records_session)
    new_dataframe.to_excel("./data/output.xlsx", index=False)

if __name__ == "__main__":
    asyncio.run(main())


import json
import os
from argparse import ArgumentParser
from tqdm import tqdm
import glob

def find_anns(anno, id):
    img_w = 640
    img_h = 640
    out = []
    for i in anno:
        img_id = i["image_id"]
        bbox = i["bbox"]
        cat = i["category_id"]
        if img_id == id:
            x = bbox[0]
            y = bbox[1]
            w = bbox[2]
            h = bbox[3]

            x_center = (x + (x + w)) / 2
            y_center = (y + (y + h)) / 2

            x_center = x_center / img_w
            y_center = y_center / img_h
            w = w / img_w
            h = h / img_h

            x_center = format(x_center, '.6f')
            y_center = format(y_center, '.6f')
            w = format(w, '.6f')
            h = format(h, '.6f')
            cat = cat - 1
            out.append([cat, x_center, y_center, w, h])
    return out

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument("-m", "--mode", type=str, default="val", help="choose the mode of created dataset if train, val, or test")
    parser.add_argument("-p", "--path", type=str, default="./challenge/annotations/instances_val.json",
                        help="path of the annotations of chosen dataset mode as json")
    parser.add_argument("-s", "--save", type=str, default="./yolov5/datasets/labels/val/",
                        help="path of images created to save as txt yolo format")
    parser.add_argument("-i", "--image", type=str, default="./yolov5/datasets/images/val/",
                        help="path of images created to save as txt yolo format")

    args = vars(parser.parse_args())
    mode = args["mode"]
    path = args["path"]
    save_path = args["save"]
    image_path = args["image"]

    imgs = glob.glob(image_path + "*.jpg")
    imgs_num = sorted([int(i.split("_")[-1][0:-4]) for i in imgs])

    if not os.path.exists(save_path):
        os.makedirs(save_path)

    with open(path, "r") as f:
        data = json.load(f)
    anno = data["annotations"]

    anno_ind = 0
    classes = ["bolt", "nut"]
    change = True
    temp = -1

    for image in tqdm(imgs_num):
        if image == temp:
            name = f"im_r_{image}.txt"
        else:
            name = f"im_{image}.txt"
        out = find_anns(anno, image)
        if out == []:
            f = open(save_path + name, "w")
        else:
            f = open(save_path + name, "w")
            for v in range(len(out)):
                if v != len(out) - 1:
                    for w in range(len(out[v])):
                        if w != len(out[v]) - 1:
                            f.write(str(out[v][w]) + " ")
                        else:
                            f.write(str(out[v][w]) + "\n")
                else:
                    for w in range(len(out[v])):
                        if w != len(out[v]) - 1:
                            f.write(str(out[v][w]) + " ")
                        else:
                            f.write(str(out[v][w]))
        anno_ind += 1
        if anno_ind > 0:
            temp = image
        f.close()


from .main import EWHandler

import threading
import logging
logger = logging.getLogger('backend')

class DataProcessThread(threading.Thread):
    """
        This thread processes the Data from track streams set as inputs using ML/DL algorithms
    """

    def __init__(self, debug=False, use_deep=False, mix=False, display_only=True):
        """ Initiates the Thread

        :param debug: (optional) (default False) a kwarg to set the debug mode
        :param use_deep: (optional) (default False) a kwarg to know if simulation should use deep learning
        :param mix: (optional) (default False) a kwarg to know if simulation should mix the use deep learning and db_scan clustering
        """
        self.mix = mix
        self.debug = debug
        self.use_deep = use_deep
        self.display_only = display_only

        self.handler = EWHandler()
        threading.Thread.__init__(self)

    def set_track_streams(self, *args):
        """
        :param *args: an unpacked list of all TrackStream objects to analyse
        """
        self.track_streams = args

    def set_sender_function(self, sender_function):
        """ Sets the function used by the backend to send data to the frontend

        :param sender_function: the function used by the backend to send data
        """
        self.sender_function = sender_function
        self.handler.set_sender_function(sender_function)

    def set_deep(self, deep):
        """ Changes the value of the use_deep attribute

        :param deep: A boolean, True if simulation should use deep learning, False if not
        """
        self.use_deep = deep

    def set_mix(self, mix):
        """ Changes the value of the mix attribute

        :param mix: A boolean, True if simulation should mix the use deep learning and db_scan clustering, False if not
        """
        self.mix = mix

    def set_display_only(self, display_only):
        """ Changes the value of the display_only attribute

        :param display_only: A boolean, True if simulation should not do any clustering, False if otherwise
        """
        self.display_only = display_only

    def run(self):
        """ Runs the given thread.
        """
        self.handler.main(*self.track_streams, debug=self.debug,
                          use_deep=self.use_deep, mix=self.mix, display_only=self.display_only)
        logger.warning('EWHandler Finished !')
        self.handler = EWHandler()

    def stop_thread(self):
        """
        Stops the thread
        """
        try:
            logger.warning("EWHandler got stop order.")
            self.handler.stop()
            logger.warning("EWHandler stopped.")
            return True
        except Exception as err:
            logger.error(err)
            return False

    def pause(self):
        """
        Pauses the thread (goes into a loop, until unpaused)
        """
        try:
            logger.warning("EWHandler got pause order.")
            self.handler.pause()
            logger.warning("EWHandler paused.")
            return True
        except Exception as err:
            logger.error(err)
            return False

    def play(self):
        """
        Unpauses the thread (leaves the loop)
        """
        try:
            logger.warning("EWHandler got play order.")
            self.handler.play()
            logger.warning("EWHandler restarting.")
            return True
        except Exception as err:
            logger.error(err)
            return False


import os
import stat

from collections import namedtuple
from glob import glob

from dlmanager import fs

File = namedtuple('File', ('path', 'stat'))

class PersistLimit(object):
    """
    Keep a list of files, removing the oldest ones when the size_limit
    is reached.

    The access time of a file is used to determine the oldests, e.g. the
    last time a file was read.

    :param size_limit: the size limit in bytes. A value of 0 means no limit.
    :param file_limit: even if the size limit is reached, this force
                       to keep at least *file_limit* files.
    """
    def __init__(self, size_limit, file_limit=5):
        self.size_limit = size_limit
        self.file_limit = file_limit
        self.files = []
        self._files_size = 0

    def register_file(self, path):
        """
        register a single file.
        """
        try:
            fstat = os.stat(path)
        except OSError:
            # file do not exists probably, just skip it
            # note this happen when backgound files are canceled
            return
        if stat.S_ISREG(fstat.st_mode):
            self.files.append(File(path=path, stat=fstat))
            self._files_size += fstat.st_size

    def register_dir_content(self, directory, pattern="*"):
        """
        Register every files in a directory that match *pattern*.
        """
        for path in glob(os.path.join(directory, pattern)):
            self.register_file(path)

    def remove_old_files(self):
        """
        remove oldest registered files.
        """
        if self.size_limit <= 0 or self.file_limit <= 0:
            return
        # sort by creation time, oldest first
        files = sorted(self.files, key=lambda f: f.stat.st_atime)
        while len(files) > self.file_limit and \
                self._files_size >= self.size_limit:
            f = files.pop(0)
            fs.remove(f.path)
            self._files_size -= f.stat.st_size
        self.files = files


#!/usr/bin/env python

from setuptools import setup

VERSION = '0.1.0'
DESC = open('README.rst').read()

setup(
    name = 'sturl',
    packages = ['sturl'],
    version = VERSION,
    description = 'URLs for humans',
    long_description = DESC,
    author = 'Adrian Ghizaru',
    author_email = 'adrian.ghizaru@gmail.com',
    url = 'https://github.com/aGHz/sturl',
    license = 'MIT',

    install_requires = [
    ],

    tests_require = ['nose', 'rednose', ],
    test_suite = 'nose.collector',

    zip_safe = False,
    include_package_data = True,
    package_data = {'': ['LICENSE', 'README.rst']},

    classifiers = [
        'Development Status :: 3 - Alpha',
        #'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Topic :: Internet',
        'Topic :: Internet :: WWW/HTTP',
    ],
    keywords = ['url'],

    download_url = 'https://github.com/aGHz/sturl/tarball/{v}'.format(v=VERSION)
)


import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from wordcloud import WordCloud
import re
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time
from bs4 import BeautifulSoup

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
url = 'https://getliner.com/user-profile/7203684' # 지속적으로 url 바뀌는 거 확인
driver.get(url)

time.sleep(1)

html = driver.page_source
soup = BeautifulSoup(html, 'html.parser')
sec = soup.select_one('section.css-auc73u')
tags = sec.select('li.css-zr54ay')
driver.quit()

collections = {}
for tag in tags:
    attr = tag['data-folder-data']
    name = re.findall('"name":"(.*?)",', attr)[0]
    page = re.findall('"savedPageCount":(.*?)}', attr)[0]
    page = int(page)
    if ', ' in name:
        names = name.split(', ')
        for n in names:
            collections[n] = page
    else:
        collections[name] = page

time.sleep(1)
font = "fonts/NanumBarunGothicBold"
font_path = f"{font}.ttf"

icon = Image.open('mask_img/Apple_logo.png')
mask = Image.new("RGB", icon.size, (255, 255, 255))
mask.paste(icon, icon)
mask = np.array(mask)

word_counts = collections
wordcloud = WordCloud(font_path=font_path, background_color="black", colormap='Reds', random_state=43, max_font_size=150, min_font_size=5, max_words=5000, mask=mask).generate_from_frequencies(word_counts)

plt.figure(figsize=(6, 6))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")

plt.show(block=False)
plt.pause(3)
plt.close()
wordcloud.to_file('wordcloud_image/wordcloud2.png')


#!/usr/bin/env python
import numpy as np
import pandas as pd
import keras
import timeit
import copy
import gc
import scipy as sp
import cv2
import os
import random
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.models import Sequential
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint
from keras.models import Model
from keras.layers import LeakyReLU,concatenate
from keras.layers import Input,Dense, Dropout, Activation,LSTM,GRU,Conv2D,Flatten,MaxPooling2D,BatchNormalization
from keras.regularizers import l1,l2,l1_l2

# Definitions
data_dir = "/home/james/data/FacesKinship/"
train_dir = data_dir + "train/"

modeltype = "cnn"
batch_size=100
width=64
height=64

traindf = pd.read_csv(data_dir+"finaltrain.csv",usecols=['p1','p2','related'])

y = traindf['related']

train_steps = int(traindf.shape[0]/batch_size)

class KinImageReader(keras.utils.Sequence):

    def __init__(self,df,target,batch_size,height,width):

        self.df = df
        self.target = target
        self.width=width
        self.height=height
        self.batch_size = batch_size
        self.on_epoch_end()

    def __len__(self):
        return int(self.df.shape[0]/self.batch_size)

    #def on_epoch_end(self):
    #    'Updates indexes after each epoch'
    #    self.indexes = np.arange(0,self.y.shape[0])

    def __getitem__(self,index):

        start = index*self.batch_size
        end = (index+1)*self.batch_size

        batchy = self.target[start:end]
        image_placeholders = np.arange(start,end)

        x1 = np.array([])
        x1 = np.zeros((len(image_placeholders),self.height,self.width,3))

        x2 = np.array([])
        x2 = np.zeros((len(image_placeholders),self.height,self.width,3))

        counter = 0

        for i in image_placeholders:

            image1 = self.df.loc[i,'p1']
            image2 = self.df.loc[i,'p2']

            #read the image
            cv_image1 = cv2.imread(image1,1)
            cv_image2 = cv2.imread(image2,1)

            #resize the image
            cv_image1 = cv2.resize(cv_image1,(self.width,self.height))
            cv_image2 = cv2.resize(cv_image2,(self.width,self.height))

            # Add to array
            x1[counter] = cv_image1/255
            x2[counter] = cv_image2/255

            counter+=1

        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(batchy)
        encoded_y = encoder.transform(batchy)

        return [x1,x2],encoded_y

train_gen = KinImageReader(traindf,y,batch_size,height,width)

inputA = Input(shape=(height,width,3))
inputB = Input(shape=(height,width,3))

# Data from first part of segment
a = Conv2D(128,kernel_size=3,activation='relu')(inputA)
a = Dropout(.3)(a)
a = BatchNormalization()(a)
a = MaxPooling2D(pool_size=(2, 2))(a)

a = Conv2D(96,kernel_size=3,activation='relu')(a)
a = Dropout(.3)(a)
a = BatchNormalization()(a)

a = Conv2D(64,kernel_size=3,activation='relu')(a)
a = Dropout(.3)(a)
a = BatchNormalization()(a)

a = Conv2D(48,kernel_size=3,activation='relu')(a)
a = Dropout(.3)(a)
a = BatchNormalization()(a)

a = Conv2D(32,kernel_size=3,activation='relu')(a)
a = Dropout(.3)(a)
a = BatchNormalization()(a)

a = Conv2D(16,kernel_size=3,activation='relu')(a)
a = Dropout(.3)(a)
a = Flatten()(a)
a = Model(inputs=inputA,outputs=a)

# Data from middle part of segment
b = Conv2D(128,kernel_size=3,activation='relu')(inputB)
b = Dropout(.3)(b)
b = BatchNormalization()(b)
b = MaxPooling2D(pool_size=(2, 2))(b)

b = Conv2D(96,kernel_size=3,activation='relu')(b)
b = Dropout(.3)(b)
b = BatchNormalization()(b)

b = Conv2D(64,kernel_size=3,activation='relu')(b)
b = Dropout(.3)(b)
b = BatchNormalization()(b)

b = Conv2D(48,kernel_size=3,activation='relu')(b)
b = Dropout(.3)(b)
b = BatchNormalization()(b)

b = Conv2D(32,kernel_size=3,activation='relu')(b)
b = Dropout(.3)(b)
b = BatchNormalization()(b)

b = Conv2D(16,kernel_size=3,activation='relu')(b)
b = Dropout(.3)(b)
b = Flatten()(b)
b = Model(inputs=inputB,outputs=b)

combined = concatenate([a.output,b.output])

final = Dense(512,activation='relu')(combined)
final = Dropout(.4)(final)
final = Dense(256,activation='relu')(final)
final = Dropout(.4)(final)
final = Dense(128,activation='relu')(final)
final = Dropout(.4)(final)
final = Dense(64,activation='relu')(final)
final = Dropout(.4)(final)
final = Dense(2, activation="softmax")(final)

model = Model(inputs=[a.input,b.input], outputs=final)

optim = Adam(lr = 0.0001)

model.compile(loss='categorical_crossentropy', optimizer=optim,metrics=['acc'])
model.summary()

best_model = data_dir + modeltype + '_weights_best.hdf5'
current_model = data_dir + modeltype + '_weights_current.hdf5'

model.load_weights(best_model)

test_dir=data_dir + "test/"

testdf = pd.read_csv(data_dir + "sample_submission.csv")

x1 = np.array([])
x1 = np.zeros((testdf.shape[0],height,width,3))

x2 = np.array([])
x2 = np.zeros((testdf.shape[0],height,width,3))

for index,line in testdf.iterrows():
    images = line['img_pair']
    image1,image2 = images.split("-")
    cv_image1 = cv2.imread(test_dir + image1,1)
    cv_image2 = cv2.imread(test_dir + image2,1)

    #resize the image
    cv_image1 = cv2.resize(cv_image1,(width,height))
    cv_image2 = cv2.resize(cv_image2,(width,height))

    # Add to array
    x1[index] = cv_image1/255
    x2[index] = cv_image2/255

prediction = model.predict([x1,x2])
print(prediction)

# Find index of maximum value from 2D numpy array
#result = np.argmax(prediction,axis=1)
testdf.drop(['is_related'], axis=1)

print(prediction[:,1])
testdf['is_related'] = np.round(prediction[:,1],4)

testdf.to_csv("foo.csv",index=False)


import json
import unittest
from binascii import unhexlify

from Crypto.SelfTest.st_common import list_test_cases
from Crypto.SelfTest.loader import load_test_vectors_wycheproof

from Crypto.Util.py3compat import tobytes, bchr
from Crypto.Cipher import AES
from Crypto.Hash import SHAKE128

from Crypto.Util.strxor import strxor

def get_tag_random(tag, length):
    return SHAKE128.new(data=tobytes(tag)).read(length)

class SivTests(unittest.TestCase):

    key_256 = get_tag_random("key_256", 32)
    key_384 = get_tag_random("key_384", 48)
    key_512 = get_tag_random("key_512", 64)
    nonce_96 = get_tag_random("nonce_128", 12)
    data = get_tag_random("data", 128)

    def test_loopback_128(self):
        for key in self.key_256, self.key_384, self.key_512:
            cipher = AES.new(key, AES.MODE_SIV, nonce=self.nonce_96)
            pt = get_tag_random("plaintext", 16 * 100)
            ct, mac = cipher.encrypt_and_digest(pt)

            cipher = AES.new(key, AES.MODE_SIV, nonce=self.nonce_96)
            pt2 = cipher.decrypt_and_verify(ct, mac)
            self.assertEqual(pt, pt2)

    def test_nonce(self):
        # Deterministic encryption
        AES.new(self.key_256, AES.MODE_SIV)

        cipher = AES.new(self.key_256, AES.MODE_SIV, self.nonce_96)
        ct1, tag1 = cipher.encrypt_and_digest(self.data)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct2, tag2 = cipher.encrypt_and_digest(self.data)
        self.assertEqual(ct1 + tag1, ct2 + tag2)

    def test_nonce_must_be_bytes(self):
        self.assertRaises(TypeError, AES.new, self.key_256, AES.MODE_SIV,
                          nonce=u'test12345678')

    def test_nonce_length(self):
        # nonce can be of any length (but not empty)
        self.assertRaises(ValueError, AES.new, self.key_256, AES.MODE_SIV,
                          nonce=b"")

        for x in range(1, 128):
            cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=bchr(1) * x)
            cipher.encrypt_and_digest(b'\x01')

    def test_block_size_128(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertEqual(cipher.block_size, AES.block_size)

    def test_nonce_attribute(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertEqual(cipher.nonce, self.nonce_96)

        # By default, no nonce is randomly generated
        self.assertFalse(hasattr(AES.new(self.key_256, AES.MODE_SIV), "nonce"))

    def test_unknown_parameters(self):
        self.assertRaises(TypeError, AES.new, self.key_256, AES.MODE_SIV,
                          self.nonce_96, 7)
        self.assertRaises(TypeError, AES.new, self.key_256, AES.MODE_SIV,
                          nonce=self.nonce_96, unknown=7)

        # But some are only known by the base cipher
        # (e.g. use_aesni consumed by the AES module)
        AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96,
                use_aesni=False)

    def test_encrypt_excludes_decrypt(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.encrypt_and_digest(self.data)
        self.assertRaises(TypeError, cipher.decrypt, self.data)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.encrypt_and_digest(self.data)
        self.assertRaises(TypeError, cipher.decrypt_and_verify,
                          self.data, self.data)

    def test_data_must_be_bytes(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(TypeError, cipher.encrypt, u'test1234567890-*')

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(TypeError, cipher.decrypt_and_verify,
                          u'test1234567890-*', b"xxxx")

    def test_mac_len(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        _, mac = cipher.encrypt_and_digest(self.data)
        self.assertEqual(len(mac), 16)

    def test_invalid_mac(self):
        from Crypto.Util.strxor import strxor_c
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct, mac = cipher.encrypt_and_digest(self.data)

        invalid_mac = strxor_c(mac, 0x01)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(ValueError, cipher.decrypt_and_verify, ct,
                          invalid_mac)

    def test_hex_mac(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        mac_hex = cipher.hexdigest()
        self.assertEqual(cipher.digest(), unhexlify(mac_hex))

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.hexverify(mac_hex)

    def test_bytearray(self):

        # Encrypt
        key = bytearray(self.key_256)
        nonce = bytearray(self.nonce_96)
        data = bytearray(self.data)
        header = bytearray(self.data)

        cipher1 = AES.new(self.key_256,
                          AES.MODE_SIV,
                          nonce=self.nonce_96)
        cipher1.update(self.data)
        ct, tag = cipher1.encrypt_and_digest(self.data)

        cipher2 = AES.new(key,
                          AES.MODE_SIV,
                          nonce=nonce)
        key[:3] = b'\xFF\xFF\xFF'
        nonce[:3] = b'\xFF\xFF\xFF'
        cipher2.update(header)
        header[:3] = b'\xFF\xFF\xFF'
        ct_test, tag_test = cipher2.encrypt_and_digest(data)

        self.assertEqual(ct, ct_test)
        self.assertEqual(tag, tag_test)
        self.assertEqual(cipher1.nonce, cipher2.nonce)

        # Decrypt
        key = bytearray(self.key_256)
        nonce = bytearray(self.nonce_96)
        header = bytearray(self.data)
        ct_ba = bytearray(ct)
        tag_ba = bytearray(tag)

        cipher3 = AES.new(key,
                          AES.MODE_SIV,
                          nonce=nonce)
        key[:3] = b'\xFF\xFF\xFF'
        nonce[:3] = b'\xFF\xFF\xFF'
        cipher3.update(header)
        header[:3] = b'\xFF\xFF\xFF'
        pt_test = cipher3.decrypt_and_verify(ct_ba, tag_ba)

        self.assertEqual(self.data, pt_test)

    def test_memoryview(self):

        # Encrypt
        key = memoryview(bytearray(self.key_256))
        nonce = memoryview(bytearray(self.nonce_96))
        data = memoryview(bytearray(self.data))
        header = memoryview(bytearray(self.data))

        cipher1 = AES.new(self.key_256,
                          AES.MODE_SIV,
                          nonce=self.nonce_96)
        cipher1.update(self.data)
        ct, tag = cipher1.encrypt_and_digest(self.data)

        cipher2 = AES.new(key,
                          AES.MODE_SIV,
                          nonce=nonce)
        key[:3] = b'\xFF\xFF\xFF'
        nonce[:3] = b'\xFF\xFF\xFF'
        cipher2.update(header)
        header[:3] = b'\xFF\xFF\xFF'
        ct_test, tag_test= cipher2.encrypt_and_digest(data)

        self.assertEqual(ct, ct_test)
        self.assertEqual(tag, tag_test)
        self.assertEqual(cipher1.nonce, cipher2.nonce)

        # Decrypt
        key = memoryview(bytearray(self.key_256))
        nonce = memoryview(bytearray(self.nonce_96))
        header = memoryview(bytearray(self.data))
        ct_ba = memoryview(bytearray(ct))
        tag_ba = memoryview(bytearray(tag))

        cipher3 = AES.new(key,
                          AES.MODE_SIV,
                          nonce=nonce)
        key[:3] = b'\xFF\xFF\xFF'
        nonce[:3] = b'\xFF\xFF\xFF'
        cipher3.update(header)
        header[:3] = b'\xFF\xFF\xFF'
        pt_test = cipher3.decrypt_and_verify(ct_ba, tag_ba)

        self.assertEqual(self.data, pt_test)

    def test_output_param(self):

        pt = b'5' * 128
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct, tag = cipher.encrypt_and_digest(pt)

        output = bytearray(128)
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        res, tag_out = cipher.encrypt_and_digest(pt, output=output)
        self.assertEqual(ct, output)
        self.assertEqual(res, None)
        self.assertEqual(tag, tag_out)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        res = cipher.decrypt_and_verify(ct, tag, output=output)
        self.assertEqual(pt, output)
        self.assertEqual(res, None)

    def test_output_param_memoryview(self):

        pt = b'5' * 128
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct, tag = cipher.encrypt_and_digest(pt)

        output = memoryview(bytearray(128))
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.encrypt_and_digest(pt, output=output)
        self.assertEqual(ct, output)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.decrypt_and_verify(ct, tag, output=output)
        self.assertEqual(pt, output)

    def test_output_param_neg(self):
        LEN_PT = 128

        pt = b'5' * LEN_PT
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct, tag = cipher.encrypt_and_digest(pt)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(TypeError, cipher.encrypt_and_digest, pt, output=b'0' * LEN_PT)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(TypeError, cipher.decrypt_and_verify, ct, tag, output=b'0' * LEN_PT)

        shorter_output = bytearray(LEN_PT - 1)
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(ValueError, cipher.encrypt_and_digest, pt, output=shorter_output)
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        self.assertRaises(ValueError, cipher.decrypt_and_verify, ct, tag, output=shorter_output)

class SivFSMTests(unittest.TestCase):

    key_256 = get_tag_random("key_256", 32)
    nonce_96 = get_tag_random("nonce_96", 12)
    data = get_tag_random("data", 128)

    def test_invalid_init_encrypt(self):
        # Path INIT->ENCRYPT fails
        cipher = AES.new(self.key_256, AES.MODE_SIV,
                         nonce=self.nonce_96)
        self.assertRaises(TypeError, cipher.encrypt, b"xxx")

    def test_invalid_init_decrypt(self):
        # Path INIT->DECRYPT fails
        cipher = AES.new(self.key_256, AES.MODE_SIV,
                         nonce=self.nonce_96)
        self.assertRaises(TypeError, cipher.decrypt, b"xxx")

    def test_valid_init_update_digest_verify(self):
        # No plaintext, fixed authenticated data
        # Verify path INIT->UPDATE->DIGEST
        cipher = AES.new(self.key_256, AES.MODE_SIV,
                         nonce=self.nonce_96)
        cipher.update(self.data)
        mac = cipher.digest()

        # Verify path INIT->UPDATE->VERIFY
        cipher = AES.new(self.key_256, AES.MODE_SIV,
                         nonce=self.nonce_96)
        cipher.update(self.data)
        cipher.verify(mac)

    def test_valid_init_digest(self):
        # Verify path INIT->DIGEST
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.digest()

    def test_valid_init_verify(self):
        # Verify path INIT->VERIFY
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        mac = cipher.digest()

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.verify(mac)

    def test_valid_multiple_digest_or_verify(self):
        # Multiple calls to digest
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.update(self.data)
        first_mac = cipher.digest()
        for x in range(4):
            self.assertEqual(first_mac, cipher.digest())

        # Multiple calls to verify
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.update(self.data)
        for x in range(5):
            cipher.verify(first_mac)

    def test_valid_encrypt_and_digest_decrypt_and_verify(self):
        # encrypt_and_digest
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.update(self.data)
        ct, mac = cipher.encrypt_and_digest(self.data)

        # decrypt_and_verify
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.update(self.data)
        pt = cipher.decrypt_and_verify(ct, mac)
        self.assertEqual(self.data, pt)

    def test_invalid_multiple_encrypt_and_digest(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct, tag = cipher.encrypt_and_digest(self.data)
        self.assertRaises(TypeError, cipher.encrypt_and_digest, b'')

    def test_invalid_multiple_decrypt_and_verify(self):
        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        ct, tag = cipher.encrypt_and_digest(self.data)

        cipher = AES.new(self.key_256, AES.MODE_SIV, nonce=self.nonce_96)
        cipher.decrypt_and_verify(ct, tag)
        self.assertRaises(TypeError, cipher.decrypt_and_verify, ct, tag)

def transform(tv):
    new_tv = [[unhexlify(x) for x in tv[0].split("-")]]
    new_tv += [ unhexlify(x) for x in tv[1:5]]
    if tv[5]:
        nonce = unhexlify(tv[5])
    else:
        nonce = None
    new_tv += [ nonce ]
    return new_tv

class TestVectors(unittest.TestCase):
    """Class exercising the SIV test vectors found in RFC5297"""

    # This is a list of tuples with 5 items:
    #
    #  1. Header + '|' + plaintext
    #  2. Header + '|' + ciphertext + '|' + MAC
    #  3. AES-128 key
    #  4. Description
    #  5. Dictionary of parameters to be passed to AES.new().
    #     It must include the nonce.
    #
    #  A "Header" is a dash ('-') separated sequece of components.
    #
    test_vectors_hex = [
      (
        '101112131415161718191a1b1c1d1e1f2021222324252627',
        '112233445566778899aabbccddee',
        '40c02b9690c4dc04daef7f6afe5c',
        '85632d07c6e8f37f950acd320a2ecc93',
        'fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff',
        None
      ),
      (
        '00112233445566778899aabbccddeeffdeaddadadeaddadaffeeddccbbaa9988' +
        '7766554433221100-102030405060708090a0',
        '7468697320697320736f6d6520706c61696e7465787420746f20656e63727970' +
        '74207573696e67205349562d414553',
        'cb900f2fddbe404326601965c889bf17dba77ceb094fa663b7a3f748ba8af829' +
        'ea64ad544a272e9c485b62a3fd5c0d',
        '7bdb6e3b432667eb06f4d14bff2fbd0f',
        '7f7e7d7c7b7a79787776757473727170404142434445464748494a4b4c4d4e4f',
        '09f911029d74e35bd84156c5635688c0'
      ),
    ]

    test_vectors = [ transform(tv) for tv in test_vectors_hex ]

    def runTest(self):
        for assoc_data, pt, ct, mac, key, nonce in self.test_vectors:

            # Encrypt
            cipher = AES.new(key, AES.MODE_SIV, nonce=nonce)
            for x in assoc_data:
                cipher.update(x)
            ct2, mac2 = cipher.encrypt_and_digest(pt)
            self.assertEqual(ct, ct2)
            self.assertEqual(mac, mac2)

            # Decrypt
            cipher = AES.new(key, AES.MODE_SIV, nonce=nonce)
            for x in assoc_data:
                cipher.update(x)
            pt2 = cipher.decrypt_and_verify(ct, mac)
            self.assertEqual(pt, pt2)

class TestVectorsWycheproof(unittest.TestCase):

    def __init__(self):
        unittest.TestCase.__init__(self)
        self._id = "None"

    def setUp(self):
        self.tv = load_test_vectors_wycheproof(("Cipher", "wycheproof"),
                                               "aes_siv_cmac_test.json",
                                               "Wycheproof AES SIV")

    def shortDescription(self):
        return self._id

    def test_encrypt(self, tv):
        self._id = "Wycheproof Encrypt AES-SIV Test #" + str(tv.id)

        cipher = AES.new(tv.key, AES.MODE_SIV)
        cipher.update(tv.aad)
        ct, tag = cipher.encrypt_and_digest(tv.msg)
        if tv.valid:
            self.assertEqual(tag + ct, tv.ct)

    def test_decrypt(self, tv):
        self._id = "Wycheproof Decrypt AES_SIV Test #" + str(tv.id)

        cipher = AES.new(tv.key, AES.MODE_SIV)
        cipher.update(tv.aad)
        try:
            pt = cipher.decrypt_and_verify(tv.ct[16:], tv.ct[:16])
        except ValueError:
            assert not tv.valid
        else:
            assert tv.valid
            self.assertEqual(pt, tv.msg)

    def runTest(self):

        for tv in self.tv:
            self.test_encrypt(tv)
            self.test_decrypt(tv)

class TestVectorsWycheproof2(unittest.TestCase):

    def __init__(self):
        unittest.TestCase.__init__(self)
        self._id = "None"

    def setUp(self):
        self.tv = load_test_vectors_wycheproof(("Cipher", "wycheproof"),
                                               "aead_aes_siv_cmac_test.json",
                                               "Wycheproof AEAD SIV")

    def shortDescription(self):
        return self._id

    def test_encrypt(self, tv):
        self._id = "Wycheproof Encrypt AEAD-AES-SIV Test #" + str(tv.id)

        cipher = AES.new(tv.key, AES.MODE_SIV, nonce=tv.iv)
        cipher.update(tv.aad)
        ct, tag = cipher.encrypt_and_digest(tv.msg)
        if tv.valid:
            self.assertEqual(ct, tv.ct)
            self.assertEqual(tag, tv.tag)

    def test_decrypt(self, tv):
        self._id = "Wycheproof Decrypt AEAD-AES-SIV Test #" + str(tv.id)

        cipher = AES.new(tv.key, AES.MODE_SIV, nonce=tv.iv)
        cipher.update(tv.aad)
        try:
            pt = cipher.decrypt_and_verify(tv.ct, tv.tag)
        except ValueError:
            assert not tv.valid
        else:
            assert tv.valid
            self.assertEqual(pt, tv.msg)

    def runTest(self):

        for tv in self.tv:
            self.test_encrypt(tv)
            self.test_decrypt(tv)

def get_tests(config={}):
    wycheproof_warnings = config.get('wycheproof_warnings')

    tests = []
    tests += list_test_cases(SivTests)
    tests += list_test_cases(SivFSMTests)
    tests += [ TestVectors() ]
    tests += [ TestVectorsWycheproof() ]
    tests += [ TestVectorsWycheproof2() ]
    return tests

if __name__ == '__main__':
    suite = lambda: unittest.TestSuite(get_tests())
    unittest.main(defaultTest='suite')


#!/bin/env python
#
#  IMH2FITS -- Convert IRAF OIF images in ".imh" format to FITS
#
#  Convert a 1-D or 2-D IRAF OIF image (i.e. a ".imh" file and the associated
#  ".pix" file) to FITS.  This task will read both old V1 files written in a
#  native byte-order as well as big-endian V2 files and swap bytes as needed.
#  IMH2FITS can be used to recover old IRAF image data and convert them to
#  FITS files when
#
#     1) no IRAF installation is currently available
#     2) old IRAF image files cannot be read because of byte-swapping
#
#  Output FITS files are written to the current directory using a filename
#  derived from the ".imh" image filename.
#
#  Usage:
#         % imh2fits [<opts>] <files>
#
#     where <opts> include:
#         -d              print debug output
#         -h              this help message
#         -l              list file rather than convert
#         -s              force byte swapping
#         -v              print verbose output
#         -w              print FITS validation warning messages
#
#         <files>         a list of ".imh" file to convert
#
#     The task will automatically search for a ".pix" pixel file that has the
#     same root name as the ".imh" file.  The output FITS file will be written
#     to the current directory using the root filename and a ".fits" extension.
#
#  Examples:
#     1) Convert all ".imh" files in the current directory to FITS:
#
#         % python imh2fits -v *.imh
#
#        The '-v' flag will print some minimal information for each converted
#        file as they are processed.

from __future__ import print_function

import os
import sys
import ast
import struct
import fnmatch
import warnings
import numpy as np
import astropy.io.fits as fits
from astropy.io.fits.verify import VerifyWarning

__author__ = 'Mike Fitzpatrick <mjfitzpatrick@gmail.com>'
__version__ = '2020-07-15'      # yyyy-mm-dd

# Global options.
debug = False
verbose = False
force_swap = False
warn = False
list_only = False

# OIF image file types
V1HDR = 1                               # OIF v1 .imh file
V1PIX = 2                               # OIF v1 .pix file
V2HDR = 3                               # OIF v2 .imh file
V2PIX = 4                               # OIF v2 .pix file

# OIF image file magic strings
V1HDR_MAGIC_BIG_ENDIAN = b'\x00i\x00m\x00h\x00d\x00r'
V1PIX_MAGIC_BIG_ENDIAN = b'\x00i\x00m\x00p\x00i\x00x'
V1HDR_MAGIC_LITTLE_ENDIAN = b'i\x00m\x00h\x00d\x00r\x00'
V1PIX_MAGIC_LITTLE_ENDIAN = b'i\x00m\x00p\x00i\x00x\x00'
V2HDR_MAGIC = b'imhv2'
V2PIX_MAGIC = b'impv2'

# IRAF pixtype codes
TY_SHORT = 3                            # signed 16-bit integer
TY_INT = 4                              # signed 32-bit integer
TY_LONG = 5                             # signed 32-bit integer
TY_REAL = 6                             # 32-bit floating point
TY_DOUBLE = 7                           # 64-bit floating point
TY_USHORT = 11                          # unsigned 16-bit integer

pixtypes = {
    TY_SHORT: 'i2',                    # (native) short
    TY_USHORT: 'U2',                   # (native) unsigned short
    TY_INT: 'i4',                      # (native) int
    TY_LONG: 'i4',                     # (native) long
    TY_REAL: 'f4',                     # (native) real
    TY_DOUBLE: 'f8',                   # (native) double
}
oiftypes = {
    TY_SHORT: 'short',                 # (native) short
    TY_USHORT: 'ushort',               # (native) unsigned short
    TY_INT: 'int',                     # (native) int
    TY_LONG: 'long',                   # (native) long
    TY_REAL: 'real',                   # (native) real
    TY_DOUBLE: 'double',               # (native) double
}

# For converting to the IRAF 1/1/1980 epoch used in timestamps (approx).
EPOCH_1980 = (10 * (3600 * 24 * 365.25))

class OIFImage(object):
    '''A Class for OIF images.
    '''

    def __init__(self, hdrfile, pixfile):
        self.path = hdrfile
        self.base = os.path.basename(hdrfile)
        self.root = os.path.splitext(self.base)[0]

        self.hdrfile = OIFHeader(hdrfile)
        self.pixfile = OIFPixfile(pixfile)

    def fileType(self, data):
        '''Determine the OIF File type.  Do this for both header and
           pixel files.
        '''
        magic = data[:10]

        if magic == V1HDR_MAGIC_BIG_ENDIAN:
            return (V1HDR, sys.byteorder != 'big')
        elif magic == V1PIX_MAGIC_BIG_ENDIAN:
            return (V1PIX, sys.byteorder != 'big')

        elif magic == V1HDR_MAGIC_LITTLE_ENDIAN:
            return (V1HDR, sys.byteorder != 'little')
        elif magic == V1PIX_MAGIC_LITTLE_ENDIAN:
            return (V1PIX, sys.byteorder != 'little')

        elif magic[:5] == V2HDR_MAGIC:
            return (V2HDR, sys.byteorder == 'little')
        elif magic[:5] == V2PIX_MAGIC:
            return (V2PIX, sys.byteorder == 'little')

        return None, None

    def toFITS(self, path, verbose=False):
        '''Save the header/pixfile to a FITS image.
        '''
        # Create the FITS object
        hdu = fits.PrimaryHDU()

        # Set the pixel array.  Note this is transposed relative to the
        # normal FITS ordering.
        if self.pixfile.ndim == 1:
            hdu.data = self.pixfile.pixels
        else:
            hdu.data = self.pixfile.pixels.T

        # Set the PHU header keywords from the OIF user area.
        for key in self.hdrfile.keywords:
            if key.strip() == '':
                continue
            st = 7 if key[:7] in ['HISTORY', 'COMMENT'] else 8
            k = key[:st].strip().replace('/', '')
            if key[st+1:].count("'") == 2:
                v = "'" + key[st+1:].split("'")[1] + "'"
                c = key[key[st+1:].find('/')].strip()
            elif key.find('/') > 0:
                v = key[st+1:].split('/')[0].strip()
                c = key[st:].split('/')[1].strip()
            else:
                v = key[st+1:].strip()
                c = ''

            if debug:
                print('k :%s:  v :%s:  c :%s:' % (str(k), str(v), str(c)))

            # Determine the keyword value type so we don't write everything
            # as a string.
            if key[:7] not in ['HISTORY', 'COMMENT']:
                if v.lower() not in ['t','f','true','false']:
                    t = ast.literal_eval(v)
                    if type(t) == int:
                        v = int(v)
                    elif type(t) == float:
                        v = float(v)
                    else:
                        v = v.strip("'")

            hdu.header[k] = (v, c)

        # Add a HISTORY keyword to track back to the original file and
        # write to the requested path.
        hdu.header.add_history('New copy of ' + str(self.hdrfile.base))

        # Print some image statistics to see whether things look reasonable.
        if verbose or list_only:
            title = ''
            if self.hdrfile.title != '':
                title = self.hdrfile.title.decode('utf-8')
            if self.pixfile.ndim == 1:
                print('%20s[%d][%s]\t%s\n\t\tmean: %g  min: %g  max: %g' %
                      (path, self.pixfile.dims[0],
                      oiftypes[self.pixfile.pixtype], title,
                      np.mean(self.pixfile.pixels),
                      np.min(self.pixfile.pixels),
                      np.max(self.pixfile.pixels)))
            else:
                print('%20s[%d,%d][%s]\t%s\n\t\tmean: %g  min: %g  max: %g' %
                      (path, self.pixfile.dims[0],  self.pixfile.dims[1],
                      oiftypes[self.pixfile.pixtype], title,
                      np.mean(self.pixfile.pixels),
                      np.min(self.pixfile.pixels),
                      np.max(self.pixfile.pixels)))

        # Write out the FITS file.
        if not list_only:
            hdu.writeto(path, overwrite=True)

class OIFHeader(OIFImage):
    '''A Class for OIF image header files.
    '''

    def __init__(self, path):
        self.path = path
        self.base = os.path.basename(path)
        self.extn = os.path.splitext(self.base)[1]

        if self.extn != '.imh':
            raise Exception('Unknown image header extension')

        # Load the file contents to memory.
        with open(path, mode='rb') as fd:
            self.content = fd.read()

        # Get the file type and whether we need to swap it.
        self.type, self.swap = self.fileType(self.content)
        ifmt = '>i' if (sys.byteorder == 'little' and self.swap) else '<i'
        if force_swap:
            ifmt = '<i' if ifmt == '>i' else '>i'

        self.data = []
        self.userarea = []
        self.title = ''
        if self.type == V1HDR:
            # Parse the V1 resident disk header.  V1 images use SPP strings
            # we convert to 'normal' strings by removing the NULLs.
            self.hdr = self.content[:2052]
            self.userarea = self.content[2052:].replace(b'\x00', b'')
            self.keywords = self.userarea.split(b'\n')[:-1]
            for i, k in enumerate(self.keywords):
                self.keywords[i] = self.keywords[i].decode('utf-8')
            self.title = self.hdr[732:888].split(b'\0\0')[0].replace(b'\x00', b'')
            self.pixtype = struct.unpack(ifmt, self.hdr[16:20])[0]

            history = self.content[892:2048].replace(b'\x00', b'')
            self.history = b''.join(history.split(b'\n'))

            self.ndim = struct.unpack(ifmt, self.hdr[20:24])[0]
            self.dims = list(struct.unpack(ifmt+'iiiiii', self.hdr[24:52]))
            self.pdims = list(struct.unpack(ifmt+'iiiiii', self.hdr[52:80]))

            self.date = struct.unpack(ifmt, self.hdr[108:112])[0] + EPOCH_1980

        elif self.type == V2HDR:
            # Parse the V2 resident disk header.  V2 images use normal
            # "packed" strings.
            self.hdr = self.content[:2046]
            self.userarea = self.content[2046:].replace(b'\x00', b'')
            self.keywords = self.userarea.split(b'\n')
            for i, k in enumerate(self.keywords):
                self.keywords[i] = self.keywords[i].decode('utf-8')
            self.title = self.hdr[638:718].replace(b'\x00', b'')
            self.pixtype = struct.unpack(ifmt, self.hdr[10:14])[0]

            history = self.content[990:2048].replace(b'\x00', b'')
            self.history = b''.join(history.split(b'\n'))

            self.ndim = struct.unpack(ifmt, self.hdr[18:22])[0]
            self.dims = list(struct.unpack(ifmt+'iiiiii', self.hdr[22:50]))
            self.pdims = list(struct.unpack(ifmt+'iiiiii', self.hdr[50:78]))

            self.date = struct.unpack(ifmt, self.hdr[106:110])[0] + EPOCH_1980

        self.printHdr()

    def printHdr(self):
        if debug:
            print('==============\nhdrfile;: ' + path)
            print('type: ' + str(self.type) + '  swap: ' + str(self.swap))
            print('title: "' + str(self.title) + '"')
            print('len hdr: ' + str(len(self.hdr)))
            print('len user: ' + str(len(self.userarea)))
            print('len data: ' + str(len(self.data)))
            print('pixtype = ' + str(self.pixtype))
            print('date = ' + str(self.date))
            print('ndim = ' + str(self.ndim))
            print('dims: ' + str(self.dims) + '  phys: ' + str(self.pdims))
            print('history: "' + str(self.history) + '"')
            if verbose:
                print(self.keywords)
                print(str(type(self.keywords)))

class OIFPixfile(OIFImage):
    '''A Class for OIF image pixel files.
    '''

    def __init__(self, path):
        self.path = path
        self.base = os.path.basename(path)
        self.extn = os.path.splitext(self.base)[1]

        if self.extn != '.pix':
            raise Exception('Unknown image pixfile extension')

        # Load the file contents to memory.
        with open(path, mode='rb') as fd:
            self.content = fd.read()

        # Get the file type and whether we need to swap it.
        self.type, self.swap = self.fileType(self.content)
        ifmt = '>i' if (sys.byteorder == 'little' and self.swap) else '<i'
        if force_swap:
            ifmt = '<i' if ifmt == '>i' else '>i'

        self.data = []
        if self.type == V1PIX:
            self.hdr = self.content[:1024]
            self.swapped = self.swap
            self.pixtype = struct.unpack(ifmt, self.hdr[16:20])[0]
            self.ndim = struct.unpack(ifmt, self.hdr[20:24])[0]
            self.dims = list(struct.unpack(ifmt+'iiiiii', self.hdr[24:52]))
            self.pdims = list(struct.unpack(ifmt+'iiiiii', self.hdr[52:80]))

            self.data = self.content[1024:]

        elif self.type == V2PIX:
            self.hdr = self.content[:2048]
            self.swapped = (struct.unpack(ifmt, self.hdr[14:18])[0] == 1)
            self.pixtype = struct.unpack(ifmt, self.hdr[10:14])[0]
            self.ndim = struct.unpack(ifmt, self.hdr[18:22])[0]
            self.dims = list(struct.unpack(ifmt+'iiiiii', self.hdr[22:50]))
            self.pdims = list(struct.unpack(ifmt+'iiiiii', self.hdr[50:78]))

            self.data = self.content[2048:]

        # Convert the pixels to a Numpy array.
        self.readPixels()
        self.printHdr()

    def readPixels(self):
        '''Read the pixels from the file contents and return a Numpy array.
        '''
        if self.type == V1PIX:
            swap = '>' if (sys.byteorder == 'little' and self.swap) else '<'
        else:
            swap = '<' if (sys.byteorder == 'little' and self.swapped) else '>'
        if force_swap:
            swap = '<' if swap == '>' else '>'
        ptype = swap + pixtypes[self.pixtype]

        raw_pixels = np.frombuffer(self.data, np.dtype(ptype), -1, 0)

        # Reshape the array and extract the pixels from the physical storage.
        if self.ndim == 1:
            self.pixels = raw_pixels[:self.dims[0]]
        elif self.ndim == 2:
            phys_pixels = np.reshape(raw_pixels, (self.pdims[1], -1)).T
            self.pixels = phys_pixels[:self.dims[0], :self.dims[1]]
        else:
            raise Exception ('Error: Images >2 dimensions not supported')

        if debug:
            print('pix type: %s shape: %s' % (ptype, str(self.pixels.shape)))
            print('final min/max: %g  %g' %
                  (np.min(self.pixels), np.max(self.pixels)))

    def printHdr(self):
        if debug:
            print('==============\npixfile;: ' + path)
            print('type: ' + str(self.type) + '  swap: ' + str(self.swap))
            print('pixtype = ' + str(self.pixtype))
            print('ndim = ' + str(self.ndim))
            print('dims: ' + str(self.dims) + '  phys: ' + str(self.pdims))

def findFile(name, path):
    '''Find the first match for the named file in the specified directory path.
    '''
    for root, dirs, files in os.walk(path):
        if name in files:
            return os.path.join(root, name)

def findAll(pattern, path):
    '''Find all files matching the given pattern in the named directory.
    '''
    result = []
    for root, dirs, files in os.walk(path):
        for name in files:
            if fnmatch.fnmatch(name, pattern):
                result.append(os.path.join(root, name))
    return result

def Usage():
    '''Print a help summary.
    '''
    print('''
Usage:
       % imh2fits [<opts>] <files>

   where <opts> include:
       -d              print debug output
       -h              this help message
       -l              list file rather than convert
       -s              force byte swapping
       -v              print verbose output
       -w              print FITS validation warning messages

       <files>         a list of ".imh" file to convert

   The task will automatically search for a ".pix" pixel file that has the
   same root name as the ".imh" file.  The output FITS file will be written
   to the current directory using the root filename and a ".fits" extension.

Examples:
   1) Convert all ".imh" files in the current directory to FITS:

       % python imh2fits -v *.imh

      The '-v' flag will print some minimal information for each converted
      file as they are processed.
    ''')

############################
# Task main()
############################

if __name__ == '__main__':

    # Process commandline args.
    files = []
    for arg in sys.argv[1:]:
        if arg in ("-h", "--help"):             # Help
            Usage()
            sys.exit()
        elif arg in ("-d", "--debug"):          # debug output
            debug = True
        elif arg in ("-l", "--list"):           # list rather than convert
            list_only = True
        elif arg in ("-v", "--verbose"):        # verbose output
            verbose = True
        elif arg in ("-s", "--swap"):           # force byte-swap
            force_swap = True
        elif arg in ("-w", "--warnings"):       # print FITS warning msgs
            warn = True
        else:
            files.append(arg)

    if not warn:
        warnings.simplefilter('ignore', category=VerifyWarning)

    # If there were no files specified on the commandline, find all files in
    # and below the current directory.
    if len(files) == 0:
        files = findAll('*.imh', './')

    for f in files:
        base = os.path.basename(f)
        root = os.path.splitext(base)[0]
        extn = os.path.splitext(base)[1]

        if extn == '.imh':
            path = os.path.dirname(f)
            if path == '':
                path = './'
            pixfile = findFile(root+'.pix', path)

            if not os.path.exists(f):
                raise Exception("Header file '%s' does not exist" % f)
            if not os.path.exists(pixfile):
                raise Exception("Pixel file '%s' does not exist" % pixfile)

            # Create an OIF object from the input files.
            image = OIFImage(f, pixfile)

            # Save to a FITS file.
            image.toFITS(root+'.fits', verbose=verbose)
        else:
            continue