""" FireObj
This is the module defining fire objects used for fire tracking

FOUR LAYERS OF OBJECTS
    a. Allfires:  the class of all fire events
    b. Fire:      the class of a fire event
    c. Cluster:   the class of active fire pixel cluster (only for supporting)
    d. FirePixel: the class of an active fire pixel
"""

# a. Object - Allfires
class Allfires:
    """ Class of allfire events at a particular time step
    """

    # initilization
    def __init__(self, t):
        """ Initiate the object with current time
        Parameters
        ----------
        t : tuple, (int,int,int,str)
            the year, month, day and 'AM'|'PM'
        """

        # Allfires object has a time stamp
        import FireTime

        # self.t = FireTime.t_nb(t,nb='previous') # initialize the object at the previous time step
        self.t = t

        # Allfires object contains a dict of Fire objects with fireID as the key (will be added after reading active fire data)
        self.fires = {}

        # Initiate lists storing fire ids which will changes at the current time step
        self.fids_expanded = (
            []
        )  # a list of ids for fires with expansion at current time step
        self.fids_new = (
            []
        )  # a list of ids for all new fires formed at current time step
        self.fids_merged = (
            []
        )  # a list of ids for fires with merging at current time step
        self.fids_invalid = (
            []
        )  # a list of ids for fires invalidated at current time step

        # cumulative recordings
        self.heritages = []  # a list of fire heritage relationships (source, target)
        self.id_dict = (
            []
        )  # this list relates the list position of the fire in the allfires object to the fire id
        # (list_index, fireid) (list position can be variable when writing out only active fires)

    # properties
    @property
    def cday(self):
        """ Datetime date of current time step
        """
        from datetime import date

        return date(*self.t[:-1])

    @property
    def ampm(self):
        """ Ampm indicator of current time step
        Parameters
        ----------
        ampm : str, 'AM'|'PM'
           the ampm option calculated from t
        """
        return self.t[-1]

    @property
    def fids(self):
        """ List of fire ids
        """
        return [i for i, f in self.fires.items()]

    @property
    def number_of_fires(self):
        """ Total number of fires (active and inactive) at this time step
        """
        return len(self.fires)

    @property
    def fids_active(self):
        """ List of active fire ids
        """
        return [i for i, f in self.fires.items() if f.isactive]

    @property
    def number_of_activefires(self):
        """ Total number of active fires at this time step
        """
        return len(self.fids_active)

    @property
    def activefires(self):
        """ dict of active fires
        """
        return {i: f for i, f in self.fires.items() if f.isactive}

    @property
    def mayactivefires(self):
        """ dict of active fires and sleepers
        """
        return {i: f for i, f in self.fires.items() if (f.isactive or f.mayreactivate)}

    @property
    def deadfires(self):
        """ dict of inactive fires not going to be reactivated
        """
        return {
            i: f for i, f in self.fires.items() if not (f.isactive or f.mayreactivate)
        }

    @property
    def fids_dead(self):
        """ List of fire ids that is not going to be reactivated
        """
        return [i for i, f in self.fires.items() if not (f.isactive or f.mayreactivate)]

    @property
    def fids_sleeper(self):
        """ List of fire ids that may reactivate
        """
        return [i for i, f in self.fires.items() if f.mayreactivate]

    @property
    def number_of_sleeper(self):
        """ Total number of sleep fires at this time step
        """
        return len(self.fids_sleeper)

    @property
    def fids_valid(self):
        """ List of valid (non-invalid) fire ids
        """
        # return [self.fires[f].fireID for f in self.fires if self.fires[f].invalid is False]
        return [i for i, f in self.fires.items() if f.invalid is False]

    @property
    def number_of_validfires(self):
        """ Total number of valid fires at this time step
        """
        return len(self.fids_valid)

    @property
    def validfires(self):
        """ List of valid fires
        """
        # return [self.fires[fid] for fid in self.fids_valid]
        return {i: f for i, f in self.fires.items() if f.invalid is False}

    @property
    def fids_updated(self):
        """ List of fire id which is updated at this time step
            (expanded, new, merged, invalid)
        """
        fids_updated = list(
            set(
                self.fids_expanded
                + self.fids_new
                + self.fids_merged
                + self.fids_invalid
            )
        )
        return fids_updated

    @property
    def fids_ne(self):
        """ List of fire id which is newly formed or expanded
               at this time step
        """
        fids_ne = sorted(set(self.fids_expanded + self.fids_new))
        return fids_ne

    # functions to be run before tracking VIIRS active fire pixels at each time step
    def update_t(self, t):
        """ Update the time (cday and ampm) for the Allfire object.
        Parameters
        ----------
        t : tuple, (int,int,int,str)
            the year, month, day and 'AM'|'PM'
        """
        self.t = list(t)  # current date and ampm

    def update_t_allfires(self, t):
        """ Update the time (t) for each Fire object in the Allfire object.
        Parameters
        ----------
        t : tuple, (int,int,int,str)
            the year, month, day and 'AM'|'PM'
        """
        for i, f in self.fires.items():
            f.t = list(t)

    def reset_newpixels(self):
        """ Reset newpixels to [] for each fire.
        """
        for i, f in self.fires.items():
            f.newpixels = []

    def cleanup(self, t):
        """ Clean up Allfires obj at each time step
        - update t (for allfires and all fires)
        - clean up lists to record fire changes
        - clean up newpixels for each fire
        """
        # time updated to t
        self.update_t(t)  # update t for allfires
        self.update_t_allfires(t)  # update t

        # reset the fids used to record changes
        self.fids_expanded = (
            []
        )  # a list of ids for fires with expansion at current time step
        self.fids_new = (
            []
        )  # a list of ids for all new fires formed at current time step
        self.fids_merged = (
            []
        )  # a list of ids for fires with merging at current time step
        self.fids_invalid = (
            []
        )  # a list of ids for fires invalidated at current time step

        # reset newpixels for each fire to empty list.
        for i, f in self.fires.items():
            f.newpixels = []

    def newyear_reset(self, regnm):
        """ reset fire ids at the start of a new year
        """
        import FireIO

        # re-id all active fires
        newfires = {}
        fidmapping = []
        fids_keep = self.fids_active + self.fids_sleeper
        for i, fid in enumerate(fids_keep):
            newfires[i] = self.fires[fid]  # record new fireID and fire object
            newfires[i].fireID = i  # also update fireID attribute of fire object
            fidmapping.append((fid, i))
        self.fires = newfires

        # lastyearfires = {}
        # fidmapping = []
        # nfid = 0
        # for f in self.activefires:
        #     ofid = f.fireID
        #     f.fireID = nfid
        #     # lastyearfires.append(f)
        #     lastyearfires[nfid] = f
        #     fidmapping.append((ofid,nfid))
        #     nfid += 1
        # self.fires = lastyearfires

        # clean heritages
        self.heritages = []

        # save the mapping table
        if len(fidmapping) > 0:
            FireIO.save_newyearfidmapping(fidmapping, self.t[0], regnm)

    # functions to be run after tracking VIIRS active fire pixels at each time step
    def record_fids_change(
        self, fids_expanded=None, fids_new=None, fids_merged=None, fids_invalid=None
    ):
        """ Update the list of ids for fires with changes at current time step.
        Parameters
        ----------
        fids_expanded : list
            ids of expanded fires
        fids_new : list
            ids of new formed fires
        fids_merged : list
            ids of fires with other fires merging to them
        fids_invalid : list
            ids of fires invalidated (due to merging with other fires)
        """
        if fids_expanded:
            self.fids_expanded = fids_expanded  # expanded fires
        if fids_new:
            self.fids_new = fids_new  # new formed fires
        if fids_merged:
            self.fids_merged = fids_merged  # fires with merging with other fires
        if fids_invalid:
            self.fids_invalid = (
                fids_invalid  # fires invalidated due to merging with other fires
            )

    def invalidate_statfires(self):
        """ If pixel density of an active fire is too large, assume it's static
                fires and invalidate it.
        """
        try:
            for i, f in self.activefires.items():
                if (f.pixden > 20) & (f.farea < 20):
                    # invalidate the fire
                    f.invalid = True

                    # add the fire id into the fids_invalid list
                    self.fids_invalid.append(f.fireID)
                    print('one static fire invalidated')
        except Exception as e:
            print(e)

    # def updateLCTmax(self):
    #     ''' update Land cover type (dominant LCT for all fire pixels) for active
    #             fires that is small in size (<1000 pixels)
    #     '''
    #     import FireIO
    #     for f in self.activefires:
    #         if (f.n_pixels < 1000):
    #             # get all LCT for the fire pixels
    #             vLCT = FireIO.get_LCT(f.locs)
    #             # extract the LCT with most pixel counts
    #             LCTmax = max(set(vLCT), key = vLCT.count)
    #             f.LCTmax = LCTmax

    # def updateftypes(self):
    #     ''' update fire type and dominant LCT for active fires
    #     '''
    #     import FireFuncs
    #     for f in self.activefires:
    #         if f.n_newpixels > 0:
    #             f.ftype = FireFuncs.set_ftype(f)
    # f.set_ftype()

# b. Object - Fire
class Fire:
    """ Class of a single fire event at a particular time step
    """

    # initilization
    def __init__(self, id, t, pixels, sensor="viirs"):
        """ Initialize Fire class with active fire pixels locations and t. This
                is only called when fire clusters forming a new Fire object.
        Parameters
        ----------
        id : int
            fire id
        t : tuple, (int,int,int,str)
            the year, month, day and 'AM'|'PM'
        pixels : list (nx5)
            latitude, longitude, line, sample, and FRP values of active fire pixels
        sensor : str
            the remote sensing instrument, 'viirs' | 'modis'; no differentiation
            between SNPP and NOAA20
        """
        import FireVector

        # initialize fire id and sensor
        self.fireID = id
        self.mergeid = id  # mergeid is the final fire id the current fire being merged; use current fire id at initialization
        self.sensor = sensor  #

        # initialize current time, fire start time, and fire final time
        tlist = list(t)  # convert (y,m,d,ampm) to [y,m,d,ampm]
        self.t = tlist  # current time
        self.t_st = tlist
        self.t_ed = tlist

        # initialize pixels
        # fpixels = [FirePixel((p[0],p[1]),(p[2],p[3],p[4]),tlist,id) for p in pixels]
        fpixels = pixels
        self.pixels = fpixels  # all pixels
        self.newpixels = fpixels  # new detected pixels
        # self.actpixels = fpixels   # new detected pixels of last active fire detection
        self.ignpixels = fpixels  # pixels at ignition

        # initialize hull using the pixels
        locs = [p.loc for p in fpixels]  # list of [lat,lon]
        # locs_geo = [p.loc_geo for p in fpixels]  # list of [lat,lon]
        hull = FireVector.cal_hull(locs, sensor)  # the hull from all locs
        self.hull = hull  # note fire.hull is not automatically updated (need explicit calculation if changes occur)

        # initialize the exterior pixels (pixels within the inward extbuffer of
        #    the hull; used for saving time for hull calculation of large fires)
        self.extpixels = FireVector.cal_extpixels(fpixels, hull)

        # fline of latest active timestep, used for sleeper threshold
        self.fline_prior = None

        # always set validate at initialization
        self.invalid = False

        # get and record fm1000 value at ignition (at the time of initilization)
        # self.stFM1000 = FireIO.get_stFM1000(hull,locs,tlist)

        # LCTmax
        # vLCT = FireIO.get_LCT(locs)
        # self.LCTmax = max(set(vLCT), key = vLCT.count)

    # properties
    @property
    def cday(self):
        """ Current day (datetime date)
        """
        from datetime import date

        return date(*self.t[:-1])

    @property
    def cdoy(self):
        """ Current day (datetime date)
        """
        return self.cday.timetuple().tm_yday

    @property
    def ampm(self):
        """ Current ampm flag, 'AM'|'PM'
        """
        return self.t[-1]

    @property
    def duration(self):
        """ Time difference between first and last active fire detection
        """
        import FireTime

        duration = FireTime.t_dif(self.t_st, self.t_ed)  # + 0.5
        return duration

    @property
    def t_inactive(self):
        """ Time difference between current time and the last active fire detection
        """
        import FireTime

        t_inactive = FireTime.t_dif(self.t_ed, self.t)
        return t_inactive

    @property
    def isactive(self):
        """ Fire active status
        """
        from FireConsts import maxoffdays

        # invalidated fires are always inactive
        if self.invalid:
            return False
        # otherwise, set to True if no new pixels detected for 5 consecutive days
        return self.t_inactive <= maxoffdays

    @property
    def isdead(self):
        """ Fire active status
        """
        from FireConsts import limoffdays

        # invalidated fires are always inactive
        if self.invalid:
            return False
        # otherwise, set to True if no new pixels detected for 5 consecutive days
        return self.t_inactive > limoffdays

    @property
    def mayreactivate(self):
        """ Fire sleeper status
        """
        from FireConsts import maxoffdays, limoffdays

        # invalidated fires are always inactive
        if self.invalid:
            return False
        # otherwise, set to True if no new pixels detected for 5 consecutive days
        return maxoffdays < self.t_inactive <= limoffdays

    @property
    def isignition(self):
        """ Is the current timestep the ignition?
        when start time == end time; and new pixel > 0
        """
        import FireTime

        if (
            len(self.newpixels) == 0
        ):  # in this case t_st = t_ed because t_ed has not been updated
            ign = 0
        else:
            ign = (FireTime.t_dif(self.t_st, self.t_ed) == 0) * 1
        return ign

    @property
    def locs(self):
        """ List of fire pixel locations (lat,lon)
        """
        return [p.loc for p in self.pixels]

    @property
    def locs_geo(self):
        """ List of fire pixel locations (lat,lon)
        """
        return [p.loc_geo for p in self.pixels]

    @property
    def locsMP(self):
        """ MultiPoint shape of locs
        """
        from shapely.geometry import MultiPoint

        mp = MultiPoint([(p[0], p[1]) for p in self.locs])
        return mp

    @property
    def n_pixels(self):
        """ Total number of fire pixels"""
        return len(self.pixels)

    @property
    def newlocs(self):
        """ List of new fire pixels locations (lat,lon)
        """
        return [p.loc for p in self.newpixels]

    @property
    def newlocs_geo(self):
        """ List of new fire pixels locations (lat,lon)
        """
        return [p.loc_geo for p in self.newpixels]

    @property
    def newlocsMP(self):
        """ MultiPoint shape of newlocs
        """
        from shapely.geometry import MultiPoint

        mp = MultiPoint([(p[0], p[1]) for p in self.newlocs])
        return mp

    @property
    def newpixelatts(self):
        """ List of new fire pixels attributes
        """
        return [
            (p.lon, p.lat, p.frp, p.DS, p.DT, p.datetime, p.ampm, p.sat)
            for p in self.newpixels
        ]

    @property
    def n_newpixels(self):
        """ Total number of new fire pixels
        """
        return len(self.newpixels)

    @property
    def extlocs(self):
        """ List of exterior fire pixel locations (lat,lon)
        """
        return [p.loc for p in self.extpixels]

    @property
    def extlocsMP(self):
        """ MultiPoint shape of extlocs
        """
        from shapely.geometry import MultiPoint

        mp = MultiPoint([(p[0], p[1]) for p in self.extlocs])
        return mp

    @property
    def n_extpixels(self):
        """ Total number of exterior fire pixels
        """
        return len(self.extpixels)

    @property
    def ignlocs(self):
        """ List of fire pixel locations (lat,lon) at ignition time step
        """
        return [p.loc for p in self.ignpixels]

    @property
    def ignlocsMP(self):
        """ MultiPoint shape of ignlocs
        """
        from shapely.geometry import MultiPoint

        mp = MultiPoint([(p[0], p[1]) for p in self.ignlocs])
        return mp

    @property
    def ignlocs_geo(self):
        """ List of fire pixel locations (lat,lon) at ignition time step
        """
        return [p.loc_geo for p in self.ignpixels]

    @property
    def ignlocsMP_geo(self):
        """ MultiPoint shape of ignlocs
        """
        from shapely.geometry import MultiPoint

        mp = MultiPoint([(p[0], p[1]) for p in self.ignlocs_geo])
        return mp

    @property
    def n_ignpixels(self):
        """ Total number of ignition fire pixels
        """
        return len(self.ignpixels)

    @property
    def farea(self):
        """ Fire spatail size of the fire event (km2)
        """
        from FireConsts import area_VI
        import numpy as np

        # get hull
        fhull = self.hull

        # If no hull, return area calculated from number of pixels
        if fhull is None:
            return self.n_pixels * area_VI
        # otherwise, use calConcHarea to calculate area,
        #   but no smaller than area_VI (sometimes calculated hull area is very small)
        else:
            # from pyproj import Geod
            # geod = Geod(ellps="WGS84")
            # area_cal = np.abs(geod.geometry_area_perimeter(self.hull)[0]/1e6)
            area_cal = fhull.area / 1e6
            return max(area_cal, area_VI)

            # import FireVector
            # return max(FireVector.calConcHarea(fhull),area_VI)

    # @property
    # def centroid(self):
    #     ''' Centroid of fire object (lat,lon)
    #     '''
    #     import FireClustering
    #
    #     # get hull
    #     fhull = self.hull
    #
    #     if fhull is not None: # centroid of the hull
    #         cent = (fhull.centroid.y,fhull.centroid.x)
    #     else: # when no hull, use the centroid of all pixels
    #         cent = FireClustering.cal_centroid(self.locs)
    #     return cent

    @property
    def pixden(self):
        """ Fire pixel density (number of pixels per km2 fire area)
        """
        farea = self.farea
        if farea > 0:
            return self.n_pixels / farea
        else:
            return 0

    @property
    def meanFRP(self):
        """ Mean FRP of the new fire pixels
        """
        frps = [p.frp for p in self.newpixels]
        if len(frps) > 0:
            m = sum(frps) / len(frps)
        else:
            m = 0
        return m

    @property
    def ftypename(self):
        """ Fire type name
        """
        import FireFuncs

        return FireFuncs.set_ftypename(self)

    @property
    def fperim(self):
        """ Perimeter length of fire hull
        """
        # get hull
        fhull = self.hull

        if fhull is None:  # if no hull, return zero
            perim = 0
        else:  # otherwise, use the hull length
            # from pyproj import Geod
            # geod = Geod(ellps="WGS84")
            # perim = geod.geometry_length(fhull)/1000 # in km
            perim = fhull.length / 1e3  # km
        return perim

    @property
    def flinepixels(self):
        """ List of all fire pixels near the fire perimeter (fine line pixels)
        """
        from shapely.geometry import Point, MultiLineString
        import FireVector
        from FireConsts import fpbuffer

        # get pixels of last active fire detection
        nps = self.newpixels

        # get hull
        fhull = self.hull

        if fhull is None:  # if no hull, return empty list
            return []
        else:
            try:
                # otherwise, extract the pixels nearl the hull
                # if hull is a polygon, return new pixels near the hull
                if fhull.type == "Polygon":
                    # lr = fhull.exterior.buffer(fpbuffer)
                    lr = FireVector.addbuffer(fhull.exterior, fpbuffer)
                    return [p for p in nps if lr.contains(Point(p.loc[0], p.loc[1]))]

                # if hull is a multipolygon, return new pixels near the hull
                elif fhull.type == "MultiPolygon":
                    # mlr = MultiLineString([x.exterior for x in fhull]).buffer(fpbuffer)
                    mlr = MultiLineString([x.exterior for x in fhull])
                    mlr = FireVector.addbuffer(mlr, fpbuffer)
                    return [p for p in nps if mlr.contains(Point(p.loc[0], p.loc[1]))]

            except Exception as e:
                print(e)
                return []

    @property
    def flplocs(self):
        """ List of fire line pixel locations (lat,lon)
        """
        return [p.loc for p in self.flinepixels]

    @property
    def flplocsMP(self):
        """ MultiPoint shape of flplocs
        """
        from shapely.geometry import Point
        import geopandas as gpd

        mp = [Point(p[0], p[1]) for p in self.flplocs]
        return gpd.GeoSeries(mp)

    @property
    def n_flinepixels(self):
        """ Total number of fire line pixels
        """
        return len(self.flinepixels)

    @property
    def fline(self):
        """ Active fire line MultiLineString shape (segment of fire perimeter with active fires nearby)
        """
        from shapely.geometry import MultiLineString
        from FireConsts import flbuffer, VIIRSbuf
        import FireVector

        if (
            self.n_flinepixels == 0
        ):  # this happens is last active pixels are within the fire scar
            return None

        # get fireline pixel locations, different from flplocsMP since it contains a VIIRS pixel buffer
        flinelocsMP = FireVector.doMultP(self.flplocs, VIIRSbuf)

        # get the hull
        fhull = self.hull

        # calculate the fire line
        if fhull is None:  # if no hull, return None
            return None
        else:  # otherwise, create shape of the active fire line
            if fhull.type == "MultiPolygon":
                # extract exterior of fire perimeter
                mls = MultiLineString([plg.exterior for plg in fhull])
                # return the part which intersects with  bufferred flinelocsMP
                # return mls.intersection(flinelocsMP.buffer(flbuffer))
                flinelocsMP_buf = FireVector.addbuffer(flinelocsMP, flbuffer)
                fline = mls.intersection(flinelocsMP_buf)

            elif fhull.type == "Polygon":
                mls = fhull.exterior
                # return mls.intersection(flinelocsMP.buffer(flbuffer))
                flinelocsMP_buf = FireVector.addbuffer(flinelocsMP, flbuffer)
                fline = mls.intersection(flinelocsMP_buf)
            else:  # if fhull type is not 'MultiPolygon' or 'Polygon', return flinelocsMP
                fline = flinelocsMP

            # we save the fire line to a new property (this is only updated when fline not None)
            self.fline_prior = fline

            return fline

    @property
    def flinelen(self):
        """ The length of active fire line
        """
        from pyproj import Geod

        try:
            flinelen = self.fline.length / 1e3  # km
            # geod = Geod(ellps="WGS84")
            # flinelen = geod.geometry_length(self.fline)/1000 # in km
        except:
            flinelen = 0

        return flinelen

    # functions
    def updateftype(self):
        """ Update fire type
        # do not use ftype as property since it may mess up when t updates (without pixel addition)
        """
        import FireFuncs

        self.ftype = FireFuncs.set_ftype(self)

    def updatefhull(self, newlocs):
        """ Update the hull using old hull and new locs
        """
        import FireVector

        hull = FireVector.cal_hull(newlocs, sensor=self.sensor)
        # use the union to include hull in past time step
        phull = self.hull
        self.hull = phull.union(hull)

    def updateextpixels(self, newpixels):
        """ Update the external pixels
        """
        import FireVector

        pextpixels = self.extpixels
        self.extpixels = FireVector.cal_extpixels(pextpixels + newpixels, self.hull)

# c. Object - Cluster
class Cluster:
    """ class of active fire pixel cluster at a particular time
    """

    # initilization
    def __init__(self, id, pixels, t, sensor="viirs"):
        """ initilization

        Parameters
        ----------
        id : int
            cluster id number
        pixels : 3-element list
            (lat, lon, FRP) of AF pixels
        t : tuple, (int,int,int,str)
            the year, month, day and 'AM'|'PM'
        """
        from datetime import date

        self.cday = date(*t[:-1])  # current date
        self.ampm = t[-1]  # current ampm
        self.id = id
        self.sensor = sensor
        self.pixels = pixels  # (x,y,FRP,...)

    # properties
    @property
    def locs(self):
        """ List of pixel locations (lat,lon)
        """
        return [(p.x, p.y) for p in self.pixels]

    @property
    def centroid(self):
        """ Centroid of the cluster (lat, lon)
        """
        import FireClustering

        return FireClustering.cal_centroid(self.locs)

    @property
    def n_pixels(self):
        """ Number of total pixels
        """
        return len(self.pixels)

    @property
    def hull(self):
        """ Fire concave hull (alpha shape)
        """
        import FireVector

        hull = FireVector.cal_hull(self.locs, self.sensor)
        return hull

    @property
    def b_box(self):
        """ Bounding box of concave hull
        """
        b_box = self.hull.bounds
        return b_box

# d. Object - FirePixel
class FirePixel:
    """ class of an acitve fire pixel, which includes
        loc : location (lat, lon)
        atts : line & sample or viirs pixel, fire radiative power
        t : time (y,m,d,ampm) of record
        origin : the fire id originally recorded (before merging)
    """

    def __init__(self, x, y, lon, lat, frp, DS, DT, ampm, t, Sat, origin):
        self.x = x
        self.y = y
        self.lat = lat
        self.lon = lon
        self.frp = frp  # frp
        # self.t = list(t)      # (year,month,day,ampm)
        self.DS = DS
        self.DT = DT
        self.ampm = ampm
        self.datetime = t  # YYYYMMDD_HHMM
        self.sat = Sat  # satellite
        self.origin = origin  # intially assigned fire id

    @property
    def loc(self):
        return (self.x, self.y)

    @property
    def loc_geo(self):
        return (self.lon, self.lat)


# from config_node import *
import paho.mqtt.client as mqtt
import config as cfg

from siisthing import SIISThing

from hardware.servo import ServoMotor

class SIISLock(SIISThing):
    def __init__(self, name: str = "lock_1"):
        SIISThing.__init__(self, name)
        self.last_state: str = ""

        self.device: ServoMotor = ServoMotor(cfg.pin)
        self.connect()

    def on_connect(self, client: mqtt.Client, userdata, flags, rc):
        # Extend the on_connect method of SIISThing
        SIISThing.on_connect(self, client, userdata, flags, rc)
        self.client.publish(self.state_topic, payload=self.last_state, qos=1, retain=True)

    def set_state(self, state: str) -> None:
        if state == "ON":
            self.device.on()
        elif state == "OFF":
            self.device.off()
        else:
            print("Invalid state")
        self.last_state = state

    def on_message(self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage):
        # Check if this is a message that sets the lock
        if message.topic == self.set_topic or message.topic == self.scheduler_topic:
            # Read the target state
            lock: str = message.payload.decode("utf-8")
            print("Setting the lock to %s" % lock)
            self.last_state = lock
            # Echo it back
            response = lock
            client.publish(self.state_topic, response, qos=1, retain=True)
        else:
            # Pass the message down
            SIISThing.on_message(self, client, userdata, message)

    def start(self):
        self.connect()
        self.client.loop_forever()

# Start polling the files
if __name__ == "__main__":
    lock = SIISLock()
    lock.start()


from collections import defaultdict
import copy
import random
import numpy as np
import os
import shutil
from urllib.request import urlretrieve
import pandas as pd
import albumentations as A
from albumentations.pytorch import ToTensorV2
import cv2
import matplotlib.pyplot as plt
from tqdm import tqdm
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.optim
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
import torchvision.models as models
from  torch.cuda.amp import autocast, GradScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.utils import resample
from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau
import timm
from timm.loss import JsdCrossEntropy
from utils import Mixup, RandAugment, AsymmetricLossSingleLabel, SCELoss, LabelSmoothingCrossEntropy, SoftTargetCrossEntropy, fmix, RAdam
from utils import merge_data, balance_data, TrainDataset, TestDataset
from PIL import Image
from torchcontrib.optim import SWA
from apex import amp

cudnn.benchmark = True
SEED = 42

def seed_everything(SEED):
    random.seed(SEED)
    os.environ['PYTHONHASHSEED'] = str(SEED)
    np.random.seed(SEED)
    torch.manual_seed(SEED)
    torch.cuda.manual_seed(SEED)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = True
seed_everything(SEED)

os.environ['CUDA_VISIBLE_DEVICES'] ="0"

def calculate_accuracy(output, target):
#     return torch.true_divide((target == output).sum(dim=0), output.size(0)).item()
    if params["mix_up"]:
        output = torch.argmax(torch.softmax(output, dim=1), dim=1)
#         return accuracy_score(output.cpu(), target.argmax(1).cpu())
        return accuracy_score(output.cpu(), target.cpu())

    output = torch.softmax(output, dim=1)
    return accuracy_score(output.argmax(1).cpu(), target.cpu())

class MetricMonitor:
    def __init__(self, float_precision=3):
        self.float_precision = float_precision
        self.reset()
        self.curr_acc = 0.
    def reset(self):
        self.metrics = defaultdict(lambda: {"val": 0, "count": 0, "avg": 0})

    def update(self, metric_name, val):
        metric = self.metrics[metric_name]

        metric["val"] += val
        metric["count"] += 1
        metric["avg"] = metric["val"] / metric["count"]
        self.curr_acc = metric["avg"]
    def __str__(self):
        return " | ".join(
            [
                "{metric_name}: {avg:.{float_precision}f}".format(
                    metric_name=metric_name, avg=metric["avg"], float_precision=self.float_precision
                )
                for (metric_name, metric) in self.metrics.items()
            ]
        )

def update_hard_sample(train_loader, model, val_criterion, thres):
    fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 6))
    train_loss_list = {'image_id':[],
                       'label':[],
                       'loss':[],
                       'fold':[]}
    model.eval()
    stream = tqdm(train_loader)
    with torch.no_grad():
        for i, (images, target, name) in enumerate(stream, start=1):
            images = images.to(params["device"], non_blocking=True)
            target = target.to(params["device"], non_blocking=True)#.view(-1,params['batch_size'])
            output = model(images)
            loss = val_criterion(output, target)
            if loss > thres:
                train_loss_list['image_id'].append(name[0])
                train_loss_list['label'].append(int(target[0].cpu().numpy()))
                train_loss_list['loss'].append(loss)
                train_loss_list['fold'].append(params['fold'])
    print("Number hard samples:",len(train_loss_list["loss"]))

    return dict(image_id=train_loss_list['image_id'],
                label=train_loss_list['label'],
                fold=train_loss_list['fold'])

def declare_model(params, load_pretrained=False, weight=None):
    if "efficientnet" in params["model"]:
        model = timm.create_model(
                params["model"],
                pretrained=False,
                num_classes=params["num_classes"],
                drop_rate=params["drop_rate"],
                drop_path_rate=0.2)
    else:
        model = timm.create_model(params["model"],
                pretrained=False,
                num_classes=params["num_classes"],
                drop_rate=params["drop_rate"])
    model = model.to(params["device"])
    optimizer = torch.optim.Adam(model.parameters(), lr=params["lr"])
    # optimizer = RAdam(model.parameters(), lr=params["lr"])
    scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=params["lr_min"], last_epoch=-1)

    if params["fp16"]:
        model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)

    if params["distributed"]:
        assert ValueError("No need to implement in a single machine")
    else:
        model = torch.nn.DataParallel(model)

    if load_pretrained:
        state_dict = torch.load(weight)
        name = params["model"]
        try:
            print(f"Load pretrained model: {name} ",state_dict["preds"], state_dict["loss"])
        except:
            print(f"Load pretrained model: {name} ",state_dict["preds"])
        model.load_state_dict(state_dict["model"])
        if params["fp16"]:
            optimizer.load_state_dict(state_dict['optimizer'])
            amp.load_state_dict(state_dict['amp'])
        best_acc = state_dict["preds"]
    else:
        best_acc = 0.85

    return model, optimizer, scheduler, best_acc

def train_epoch(train_loader, model, criterion, optimizer, epoch, params):
    metric_monitor = MetricMonitor()
    model.train()
    if params["hard_negative_sample"]:
        stream = tqdm(update_train_loader)
    else:
        stream = tqdm(train_loader)
    for i, (images, target, _, _) in enumerate(stream, start=1):
#         with autocast():
        images = images.to(params["device"]) #, non_blocking=True)
        target = target.to(params["device"]) #, non_blocking=True) #.view(-1,params['batch_size'])
        if params["mix_up"]:
            images , mtarget = mixup_fn(images, target)
        if epoch > 10 and params["fmix"]:
            images , ftarget = fmix(images, target, alpha=1., decay_power=5.,
                        shape=(params["image_size"],params["image_size"]),
                        device=params["device"])

        output = model(images)
        if isinstance(output, (tuple, list)):
            output = output[0]

        if epoch > 10 and params["fmix"]:
            loss = criterion_fmix(output, ftarget[0]) * ftarget[2] + criterion_fmix(output, ftarget[1]) * (1. - ftarget[2])

        else:
            loss = criterion(output, mtarget)

#         loss = criterion(output, target)

        if params['gradient_accumulation_steps'] > 1:
            loss = loss / params['gradient_accumulation_steps']

        accuracy = calculate_accuracy(output, target)
        metric_monitor.update("Loss", loss.item())
        metric_monitor.update("Accuracy", accuracy)
        optimizer.zero_grad()
        loss.backward()
#         with amp.scale_loss(loss, optimizer) as scaled_loss:
#             scaled_loss.backward()

        optimizer.step()
        stream.set_description(
            "Epoch: {epoch}. Train.      {metric_monitor}".format(epoch=epoch, metric_monitor=metric_monitor)
        )

def validate(val_loader, model, criterion, optimizer, epoch, params, fold, best_acc):
    metric_monitor = MetricMonitor()
    model.eval()
    stream = tqdm(val_loader)
    with torch.no_grad():
        for i, (images, target, _,_) in enumerate(stream, start=1):
            images = images.to(params["device"], non_blocking=True)
            target = target.to(params["device"], non_blocking=True)#.view(-1,params['batch_size'])
            output = model(images)
            loss = val_criterion(output, target)
            output = torch.softmax(output, dim = 1)
            accuracy = accuracy_score(output.argmax(1).cpu(), target.cpu())

            stream.set_description(
                "Epoch: {epoch}. Validation. {metric_monitor}".format(epoch=epoch, metric_monitor=metric_monitor)
            )
            metric_monitor.update("Loss", loss.item())
            metric_monitor.update("Accuracy", accuracy)

        #to save weight
        if (metric_monitor.curr_acc > best_acc): # or epoch == params["epochs"]:
            print(f"Save best weight at acc {round(metric_monitor.curr_acc,4)}, epoch: {epoch}")
            best_acc = metric_monitor.curr_acc

            directory = f'weights/{params["model"]}'
            if not os.path.exists(directory):
                os.makedirs(directory)
            if params["fp16"]:
                torch.save({'model': model.state_dict(),
                    'optimizer': optimizer.state_dict(),
                    'amp': amp.state_dict(),
                    'loss': loss,
                    'preds': round(metric_monitor.curr_acc,4)},
                     f'weights/{params["model"]}/{params["model"]}_fold{fold}_best_epoch_{epoch}.pth')
            else:
                torch.save({'model': model.state_dict(),
                    'loss': loss,
                    'optimizer': optimizer.state_dict(),
                    'preds': round(metric_monitor.curr_acc,4)},
                     f'weights/{params["model"]}/{params["model"]}_fold{fold}_best_epoch_{epoch}.pth')
    return best_acc

def train_epoch_bak(train_loader, model, criterion, optimizer, epoch, params, scaler=None, criterion_fmix=None):
    metric_monitor = MetricMonitor()
    model.train()
    if params["hard_negative_sample"]:
        stream = tqdm(update_train_loader)
    else:
        stream = tqdm(train_loader)
    for i, (images, target, _, _) in enumerate(stream, start=1):
        mix_decision = np.random.rand()
        with autocast():
            images = images.to(params["device"]) #, non_blocking=True)
            target = target.to(params["device"]) #, non_blocking=True) #.view(-1,params['batch_size'])
            if params["mix_up"]:
                images , mtarget = mixup_fn(images, target)
                if params["distill_soft_label"]:
                    mtarget = mtarget*0.7 + soft_target.to(params['device']) * 0.3

            # if epoch > 15 and params["fmix"] and not params["fp16"]:
            #     images , ftarget = fmix(images, target, alpha=1., decay_power=5.,
            #                         shape=(params["image_size"],params["image_size"]),
            #                         device=params["device"])

            output = model(images)

            if isinstance(output, (tuple, list)):
                output = output[0]

            loss = criterion(output, mtarget)
            # if epoch > 70 and params["fmix"]:
                # loss = criterion_fmix(output, ftarget[0]) * ftarget[2] + criterion_fmix(output, ftarget[1]) * (1. - ftarget[2])
            # else:
                # loss = criterion(output, mtarget)
            # loss1 = symetric_criterion(output, target)
            # loss2 = asymetric_criterion(output, target)

            if params['gradient_accumulation_steps'] > 1:
                loss = loss / params['gradient_accumulation_steps']

            accuracy = calculate_accuracy(output, target)
            metric_monitor.update("Loss", loss.item())
            metric_monitor.update("Accuracy", accuracy)
            optimizer.zero_grad()
            loss.backward()
            # with amp.scale_loss(loss, optimizer) as scaled_loss:
                # scaled_loss.backward()
            optimizer.step()
            # optimizer.update_swa()
            # scaler.step(optimizer)

            stream.set_description(
                "Epoch: {epoch}. Train.      {metric_monitor}".format(epoch=epoch, metric_monitor=metric_monitor)
            )

if __name__ == "__main__":

    root = os.path.join(os.environ["HOME"], "Workspace/datasets/taiyoyuden/cassava")
    train = pd.read_csv(f'{root}/train.csv')
    train_external = pd.read_csv(f'{root}/external/train_external.csv')
    test_external = pd.read_csv(f'{root}/external/test_external.csv')
    # test_external_pseudo = pd.read_csv(f'{root}/external/test_external_pseudo_0.8.csv')
    test_external_pseudo = pd.read_csv(f'{root}/external/test_external_pseudo_0.8_round2.csv')
    test = pd.read_csv(f'{root}/sample_submission.csv')
    label_map = pd.read_json(f'{root}/label_num_to_disease_map.json', orient='index')

    models_name = ["resnest26d","resnest50d","tf_efficientnet_b3_ns" ,"tf_efficientnet_b4_ns","legacy_seresnext26_32x4d", "vit_base_patch16_384"]
    WEIGHTS = [

        # #"./weights/resnest50d/resnest50d_fold0_best_epoch_30_final_1st.pth",
        # #"./weights/resnest50d/resnest50d_fold0_best_epoch_13_final_2nd.pth",
        # "./weights/resnest50d/resnest50d_fold0_best_epoch_10_final_3rd.pth",
        # #"./weights/resnest50d/resnest50d_fold1_best_epoch_95_final_1st.pth",
        # #"./weights/resnest50d/resnest50d_fold1_best_epoch_17_final_2nd.pth",
        # "./weights/resnest50d/resnest50d_fold1_best_epoch_8_final_5th_pseudo.pth",
        # #"./weights/resnest50d/resnest50d_fold2_best_epoch_50_final_1st.pth",
        # "./weights/resnest50d/resnest50d_fold2_best_epoch_22_final_2nd.pth",
        # #"./weights/resnest50d/resnest50d_fold3_best_epoch_2_final_2nd.pth",
        # "./weights/resnest50d/resnest50d_fold3_best_epoch_1_final_3rd.pth",
        # #"./weights/resnest50d/resnest50d_fold4_best_epoch_10_final_2nd.pth",
        # #"./weights/resnest50d/resnest50d_fold4_best_epoch_15_final_3rd.pth" ,
        # #"./weights/resnest50d/resnest50d_fold4_best_epoch_10_final-4th.pth"
        # "./weights/resnest50d/resnest50d_fold4_best_epoch_1_final_5th_pseudo.pth",
        #
        # # "./weights/resnest26d/resnest26d_fold0_best_epoch_4_final_2nd.pth",
        # "weights/resnest26d/resnest26d_fold0_best_epoch_19_final_3rd.pth",
        # "./weights/resnest26d/resnest26d_fold1_best_epoch_7_final_2nd.pth",
        # "./weights/resnest26d/resnest26d_fold2_best_epoch_4_final_2nd.pth",
        # # "./weights/resnest26d/resnest26d_fold3_best_epoch_15_final_2nd.pth",
        # "weights/resnest26d/resnest26d_fold3_best_epoch_10_final_3rd.pth",
        # # "./weights/resnest26d/resnest26d_fold4_best_epoch_21_final_2nd.pth",
        # "weights/resnest26d/resnest26d_fold4_best_epoch_6_final_3rd.pth"
        #
        # "weights/tf_efficientnet_b4_ns/tf_efficientnet_b4_ns_fold0_best_epoch_75_final_1st.pth",
        # "weights/tf_efficientnet_b4_ns/tf_efficientnet_b4_ns_fold1_best_epoch_53_final_1st.pth",
        # "weights/tf_efficientnet_b4_ns/tf_efficientnet_b4_ns_fold2_best_epoch_75_final_1st.pth",
        # "weights/tf_efficientnet_b4_ns/tf_efficientnet_b4_ns_fold3_best_epoch_84_final_1st.pth",
        # "weights/tf_efficientnet_b4_ns/tf_efficientnet_b4_ns_fold4_best_epoch_66_final_1st.pth"
        "weights/tf_efficientnet_b4_ns/tf_efficientnet_b4_ns_fold1_best_epoch_65_final_1st.pth",
        "weights/legacy_seresnext26_32x4d/legacy_seresnext26_32x4d_fold2_best_epoch_92_final_1st.pth",
        "weights/legacy_seresnext26_32x4d/legacy_seresnext26_32x4d_fold2_best_epoch_28_final_2nd.pth"

    ]

    model_index = 4
    ckpt_index = -1
    params = {
        "visualize": False,
        "fold": [2],
        "distill_soft_label":False,
        "train_external": True,
        "train_clean_only": False,
        "test_external": False,
        "load_pretrained": True,
        "fp16": False,
        "resume": False,
        "image_size": 512,
        "num_classes": 5,
        "model": models_name[model_index],
        "device": "cuda",
        "lr": 1e-4,
        "lr_min":1e-7,
        "batch_size": 8,
        "num_workers": 8,
        "epochs": 30,
        "gradient_accumulation_steps": 1,
        "drop_block": 0.2,
        "drop_rate": 0.2,
        "mix_up": True,
        "cutmix":True,
        "fmix":True,
        "smooth_label": 0.1,
        "rand_aug": False,
        "local_rank":0,
        "distributed": False,
        "hard_negative_sample": False,
        "tta": True,
        "train_phase":True,
        "balance_data":False,
        "kfold_pred":False
    }
    scaler = GradScaler()

    train_transform = A.Compose(
        [
            A.RandomResizedCrop(height=params["image_size"], width=params["image_size"], p=1),
            A.OneOf([
                A.RandomRotate90(p=0.5),
                A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=0.5),], p=1.
            ),
    #         A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=0.5),
            A.HorizontalFlip(p=0.5),
            A.VerticalFlip(p=0.5),
            A.IAAAffine(rotate=0.2, shear=0.2,p=0.5),
            A.CoarseDropout(max_holes=20, max_height=int(params["image_size"]/15), max_width=int(params["image_size"]/15), p=0.5),
    #         A.IAAAdditiveGaussianNoise(p=1.),
            A.MedianBlur(p=0.5),
            A.Equalize(p=0.2),
            A.GridDistortion(p=0.2),
    #         A.RandomGridShuffle(grid=(100, 100), p=0.5),
            A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
            ToTensorV2(),
        ]
    )
    val_transform = A.Compose(
        [
            A.CenterCrop(height=params["image_size"], width=params["image_size"], p=1),
            A.Resize(params["image_size"],params["image_size"]),
            A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
            ToTensorV2(),
        ]
    )
    if params["cutmix"]:
        mixup_fn = Mixup(mixup_alpha=1., cutmix_alpha=1., label_smoothing=params["smooth_label"], num_classes=params["num_classes"])
    else:
        mixup_fn = Mixup(mixup_alpha=1., label_smoothing=params["smooth_label"], num_classes=params["num_classes"])

    val_criterion = nn.CrossEntropyLoss().to(params["device"])
    criterion_fmix = nn.CrossEntropyLoss().to(params["device"])
    criterion = LabelSmoothingCrossEntropy().to(params["device"])
    if params["mix_up"]:
        criterion = SoftTargetCrossEntropy().to(params["device"])
    asymetric_criterion = AsymmetricLossSingleLabel().to(params["device"])
    symetric_criterion = SCELoss(smooth_label=params["smooth_label"]).to(params["device"])

     # model = getattr(models, params["model"])(pretrained=False, num_classes=5)
    folds = train.copy()
    Fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
    for n, (train_index, val_index) in enumerate(Fold.split(folds, folds['label'])):
        folds.loc[val_index, 'fold'] = int(n)
    folds['fold'] = folds['fold'].astype(int)

    for i, fold_idx in enumerate(params["fold"]):
        print(f"Train Fold: {fold_idx}")
        # dataset train/test split
        fold = fold_idx
        train_idx = folds[folds['fold'] != fold].index
        val_idx = folds[folds['fold'] == fold].index

        train_folds = folds.loc[train_idx].reset_index(drop=True)
        val_folds = folds.loc[val_idx].reset_index(drop=True)

        for idx,label in enumerate(train_external['label']):
            train_external.loc[idx,'fold'] = fold
        train_external['fold'] = train_external['fold'].astype(int)

        if params["train_external"]:
            train_folds = merge_data(train_folds, train_external)
            params["distill_soft_label"] = False

        if params["train_clean_only"]:
            train_folds = train_external
            params["distill_soft_label"] = False

        if params["test_external"]:
            train_folds = merge_data(train_folds, test_external_pseudo)
            params["distill_soft_label"] = False

        if params["balance_data"]:
            train_folds = balance_data(train_folds, mode="undersampling")
            val_folds = balance_data(val_folds, mode="undersampling", val=True)
            params["distill_soft_label"] = False

        # for distillation using soft label
        if params["distill_soft_label"]:
            soft_target_list = set(params["fold"]) - set([fold_idx])
            for k, f in enumerate(soft_target_list):
                if k == 0:
                    soft_target_distill = pd.read_csv(f'./error_analysis/val_{params["model"]}_{f}_pred.csv')
                else:
                    soft_target_distill = merge_data(soft_target_distill,  pd.read_csv(f'./error_analysis/val_{params["model"]}_{f}_pred.csv'))
            soft_target_distill = soft_target_distill.reset_index(drop=True)
            soft_target_distill = soft_target_distill.set_index('image_id').sort_index().values
            train_dataset = TrainDataset(train_folds, root, transform=train_transform, soft_df=soft_target_distill)
        else:
            train_dataset = TrainDataset(train_folds, root, transform=train_transform)
        val_dataset = TrainDataset(val_folds, root, transform=val_transform)

        if params["hard_negative_sample"]:
            train_loader = DataLoader(
                train_dataset, batch_size=1, shuffle=True, num_workers=params["num_workers"], pin_memory=True,
            )
        else:
            train_loader = DataLoader(
                train_dataset, batch_size=params["batch_size"], shuffle=True, num_workers=params["num_workers"], pin_memory=True,
            )
        val_loader = DataLoader(
            val_dataset, batch_size=params["batch_size"], shuffle=False, num_workers=params["num_workers"], pin_memory=True,
        )

        # model declaration
        if "efficientnet" in params["model"]:
            model = timm.create_model(
                params["model"],
                pretrained=True,
                num_classes=params["num_classes"],
                drop_rate=params["drop_rate"],
                drop_path_rate=0.3)
        else:
            model = timm.create_model(
                params["model"],
                pretrained=True,
                num_classes=params["num_classes"])
                #drop_block_rate=params["drop_block"])

        model = model.to(params["device"])
        optimizer = torch.optim.Adam(model.parameters(), lr=params["lr"])
        # scheduler = CosineAnnealingLR(optimizer, T_max=10, eta_min=params["lr_min"], last_epoch=-1)
        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=params["lr_min"], last_epoch=-1)

        if params["fp16"]:
            model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)

        if params["distributed"]:
            assert ValueError("No need to implement in a single machine")
        else:
            model = torch.nn.DataParallel(model)
        if params["load_pretrained"]:
            state_dict = torch.load(WEIGHTS[ckpt_index])
            print("Load pretrained model: ",state_dict["preds"])
            model.load_state_dict(state_dict["model"])
            if params["fp16"]:
                optimizer.load_state_dict(state_dict['optimizer'])
                amp.load_state_dict(state_dict['amp'])

            best_acc = state_dict["preds"]
            # Hard negative mining based on train data and pretrained model on that data
            if params["hard_negative_sample"]:
                update_train_data = update_hard_sample(train_loader, model, val_criterion, thres=0.2)
                update_train_folds = pd.DataFrame(data=update_train_data)
                update_train_folds = pd.concat(5*[update_train_folds])
                #check the update distribution when filter data
                print("Class distribution for the new data")
                visualize_class_dis(update_train_folds, params["fold"])

                #update the training set
                update_train_dataset = TrainDataset(update_train_folds, transform=train_transform)
                update_train_loader = DataLoader(
                    update_train_dataset, batch_size=params["batch_size"], shuffle=True, num_workers=params["num_workers"], pin_memory=True,
                )
        else:
            best_acc = 0.83

        # trainning process
        for epoch in range(1, params["epochs"] + 1):
            train_epoch(train_loader, model, criterion, optimizer, epoch, params)
            best_acc = validate(val_loader, model, criterion, optimizer ,epoch, params, fold, best_acc)

        del model


import argparse
import sys

import related_person
from related_person import Gender, RelatedPerson, RelatedPersonError

class RawPersonRecord(object):
    FIELDS = 5
    def __init__(self, id, name, father_id, mother_id, gender, row):
        self.id = id
        self.name = name
        self.father_id = father_id
        self.mother_id = mother_id
        self.gender = gender
        self.row = row

    @staticmethod
    def make_from_line(line, row):
        l = line.strip().split('\t')
        if len(l) != RawPersonRecord.FIELDS:
            raise ValueError("row {}: has {} fields, not {}".format(row, len(l), RawPersonRecord.FIELDS))
        t = tuple(l+[row])
        return RawPersonRecord(*t)

class LoadPeople(object):
    NULL_ID = '0'

    def __init__(self):
        self.buffer = []
        self.people_index = {}

    @staticmethod
    def all_people(people_index):
        print()
        for id in sorted(people_index.keys()):
            print(str(people_index[id]))

    def phase1(self):
        parser = argparse.ArgumentParser(description="Load related people from a file")
        parser.add_argument('--log_file', '-l', type=str, help="File for logging data")
        parser.add_argument('-p', '--people_data', required=True, type=argparse.FileType('r'),
            help="File with related people, tab delimited; can be provided on stdin")
        return parser.parse_args()

    def phase2(self, people_data):
        # Phase 2:
        row = 0
        errors = []
        bad_ids = set()
        for line in people_data:
            row += 1
            try:
                self.buffer.append(RawPersonRecord.make_from_line(line, row))
            except ValueError as e:
                errors.append(str(e))

        # check IDs, genders, & create RelatedPersons
        for raw_person in self.buffer:
            try:
                # check for dupes
                if raw_person.id in self.people_index:
                    bad_ids.add(raw_person.id)
                    del self.people_index[raw_person.id]
                if raw_person.id in bad_ids:
                    raise RelatedPersonError("duplicate ID: {}".format(raw_person.id))

                # get and check gender
                gender = Gender.get_gender(raw_person.gender)

                # make RelatedPerson
                related_person = RelatedPerson(raw_person.id, raw_person.name, gender)
                self.people_index[raw_person.id] = related_person
            except RelatedPersonError as e:
                errors.append("row {}: {}".format(raw_person.row, str(e)))

        if errors:
            sys.stderr.write('\n- individual errors -\n')
            sys.stderr.write('\n'.join(errors))
            sys.stderr.write('\n')

    def check_parent(self, raw_person, parent):
        if parent == 'mother':
            if raw_person.mother_id != LoadPeople.NULL_ID:
                if raw_person.mother_id not in self.people_index:
                    raise RelatedPersonError("{} missing mother {}".format(raw_person.id, raw_person.mother_id))
        elif parent == 'father':
            if raw_person.father_id != LoadPeople.NULL_ID:
                if raw_person.father_id not in self.people_index:
                    raise RelatedPersonError("{} missing father {}".format(raw_person.id, raw_person.father_id))

    def set_parent(self, raw_person, parent):
        related_person = self.people_index[raw_person.id]
        if parent == 'mother':
            if raw_person.mother_id != LoadPeople.NULL_ID:
                mother = self.people_index[raw_person.mother_id]
                related_person.set_mother(mother)
        elif parent == 'father':
            if raw_person.father_id != LoadPeople.NULL_ID:
                father = self.people_index[raw_person.father_id]
                related_person.set_father(father)

    def phase3(self):
        # Phase 3:
        errors = []
        bad_ids = set()

        for raw_person in self.buffer:
            if raw_person.id in self.people_index:

                # check existence of parents
                for parent in ['mother', 'father']:
                    try:
                        self.check_parent(raw_person, parent)
                    except RelatedPersonError as e:
                        errors.append("row {}: {}".format(raw_person.row, str(e)))
                        bad_ids.add(raw_person.id)

                # set parents, which checks their gender
                if raw_person.id not in bad_ids:
                    for parent in ['mother', 'father']:
                        try:
                            self.set_parent(raw_person, parent)
                        except RelatedPersonError as e:
                            errors.append("row {}: for {} {}".format(raw_person.row, raw_person.id, str(e)))
                            bad_ids.add(raw_person.id)

        # delete all the bad people
        for bad_id in bad_ids:
            del self.people_index[bad_id]

        if errors:
            sys.stderr.write('\n- relatedness errors -\n')
            sys.stderr.write('\n'.join(errors))
            sys.stderr.write('\n')

    def main(self):
        args = self.phase1()
        self.phase2(args.people_data)
        self.phase3()
        self.all_people(self.people_index)

LoadPeople().main()


# Mostly Stolen From CS230.
# `https://github.com/cs230-stanford/cs230-code-examples/blob/master/pytorch/vision/utils.py`
# Utility Functions for logging and Saving model.
import os
import torch
import numpy as np
'''
 ##################
 # Example Usage  #
 ##################
state = {
    'epoch': epoch + 1,
    'state_dict': model.state_dict(),
    'optim_dict' : optimizer.state_dict()
}
utils.save_checkpoint(
    state,
    checkpoint=model_dir
)
'''

def save_checkpoint(state, checkpoint='../Data/FullData', name='best_model.tar'):
    """
    Saves the best model and training parameters at
    `checkpoint + 'best_model.tar'``.
    Args:
        state:
            (dict) contains model's state_dict,
            may contain other keys such as epoch, optimizer state_dict
        checkpoint:
            (string) folder where parameters are to be saved
    """
    filepath = os.path.join(checkpoint, name)
    if not os.path.exists(checkpoint):
        print("Checkpoint Directory does not exist! Making directory {}".format(checkpoint))
        os.mkdir(checkpoint)
    torch.save(state, filepath)

def load_checkpoint(checkpoint, model, optimizer=None):
    """
    Loads model parameters (state_dict) from file_path.
    If optimizer is provided, loads state_dict of
    optimizer assuming it is present in checkpoint.
    Args:
        checkpoint:
            (string) filename which needs to be loaded
        model:
            (torch.nn.Module) model for which the parameters are loaded
        optimizer:
            (torch.optim) optional: resume optimizer from checkpoint
    """
    if not os.path.exists(checkpoint):
        raise("File doesn't exist {}".format(checkpoint))
    checkpoint = torch.load(
        checkpoint,
        map_location=lambda storage, loc: storage
    )
    model.load_state_dict(checkpoint['state_dict'])

    if optimizer:
        optimizer.load_state_dict(checkpoint['optim_dict'])

class SubtructMeanImage(object):
    '''
        subtract the mean image from the image
    '''
    def __init__(self, path_to_mean_image):
        self.mean_image = torch.from_numpy(
            np.load(path_to_mean_image)
        ).permute(2, 0, 1).float()
    def __call__(self, image_tensor):
        image =  image_tensor - self.mean_image
        return image

def show_image(pytorch_tensor):
    import torchvision.transforms.functional as F
    a = F.to_pil_image(pytorch_tensor)
    return a


from brownie import (
    accounts,
    exceptions,
    UnstoppableLender,
    DamnValuableToken,
    ReceiverUnstoppable,
)
from web3 import Web3
import pytest

# Pool has 1M * 10**18 tokens
TOKENS_IN_POOL = Web3.toWei("1000000", "ether")
INITIAL_ATTACKER_BALANCE = Web3.toWei("100", "ether")

def before():
    # Setup scenario
    deployer = accounts[0]
    attacker = accounts[1]
    randomUser = accounts[2]
    dvt_token = DamnValuableToken.deploy({"from": deployer})
    lender = UnstoppableLender.deploy(dvt_token.address, {"from": deployer})

    # approve lender for deployer
    dvt_token.approve(lender.address, TOKENS_IN_POOL, {"from": deployer})
    # setup lender with 1,000,000 DVT
    lender.depositTokens(TOKENS_IN_POOL, {"from": deployer})
    # setup attacker with 100 DVT
    dvt_token.transfer(attacker.address, INITIAL_ATTACKER_BALANCE, {"from": deployer})

    # assert balances and that users can execute flash loans.
    assert dvt_token.balanceOf(lender.address) == TOKENS_IN_POOL
    assert dvt_token.balanceOf(attacker.address) == INITIAL_ATTACKER_BALANCE
    receiver = ReceiverUnstoppable.deploy(lender.address, {"from": randomUser})
    receiver.executeFlashLoan(10, {"from": randomUser})

def run_exploit():
    # remove pass and add exploit code here
    # attacker = accounts[1] - account to be used for exploit
    pass

def after():
    randomUser = accounts[2]
    # Confirm other users now cannot execute flash loans
    with pytest.raises(exceptions.VirtualMachineError):
        ReceiverUnstoppable[-1].executeFlashLoan(10, {"from": randomUser})

def test_unstoppable_challenge():
    before()
    run_exploit()
    after()


import pygame

class Output:
    def __init__(self):
        self.scorenum = 0
        self.score = 0
        self.count = 3
        self.img = pygame.image.load('images/live.png')
        self.img = pygame.transform.scale(self.img, (70, 70))
        self.dead = 0 # 1 / 0

    def show(self, screen):
        if self.count > 0:
            font = pygame.font.SysFont('Helvetica', 30, True)
            s = self.scorenum
            ScoreRender = font.render('SCORE:{}'.format(s), False, (255, 255, 255))
            for i in range(self.count):
                screen.blit(self.img, (3 + i * 32, 50))
            screen.blit(ScoreRender, (13, 13))
        else:
            font = pygame.font.SysFont('Helvetica', 20, True)
            screen.blit(font.render('GAME    OVER!', False, (255, 0, 0)), (330, 353))
            pygame.display.flip()
            over = False
            cnt = 0
            while not over:
                for event in pygame.event.get():  # Получение всех событий
                    if event.type == pygame.QUIT:  # Событие выхода
                        over = True
                if cnt == 4000:
                    over = True
                pygame.time.wait(1)
                cnt += 1
            #time.sleep(5)
            self.dead = 1

    def seed_score(self):
        self.scorenum += 10
        self.score += 10

    def bigseed_score(self):
        self.scorenum += 50
        self.score += 50

    def berry_score(self):
        self.scorenum += 100

    def ghost(self, n):
        self.scorenum += 100*2**n
        self.score += 100*2**n

    def death(self):
        self.count = self.count - 1


from afbench import Benchmark, BenchmarkInfo

import numpy as np
import matplotlib.pyplot as plt

fft_info = BenchmarkInfo('benchmarks/fft.json')
print('Available benchmarks: {}'.format(fft_info.benchmark_names))
for bench in fft_info.benchmark_names:
    print('{} parameters:'.format(bench))
    for param in fft_info.params(bench):
        print('{}: {}'.format(param, fft_info.paramvals(bench, 'f32', param)))

fft1 = Benchmark(
    filepath='benchmarks/fft.json',
    name='fft1',
    filters={
        'dim1': 1,
        'dim2': 1,
        'fft_dim': 1
    }
)
fft1_x_vals = fft1.collect_param_vals('dim0', 'f32')
fft1_y_vals = fft1.collect_real_times('f32')

fft2 = Benchmark(
    filepath='benchmarks/fft.json',
    name='fft2',
    filters={
        'dim1': 16,
        'dim2': 1,
        'fft_dim': 2
    }
)
fft2_x_vals = fft2.collect_param_vals('dim0', 'f32')
for i in range(len(fft2_x_vals)):
    fft2_x_vals[i] *= 16
fft2_y_vals = fft2.collect_real_times('f32')

np_fft1_x = np.asarray(fft1_x_vals)
np_fft1_y = np.asarray(fft1_y_vals)
np_fft2_x = np.asarray(fft2_x_vals)
np_fft2_y = np.asarray(fft2_y_vals)

plt.plot(np_fft1_x, np_fft1_y, 'bo-',
         np_fft2_x, np_fft2_y, 'go-'
)
plt.xscale('log')
plt.yscale('log')
plt.show()


import re
import base64
import json
import datetime
from io import BytesIO
from xml.etree import ElementTree
from PyPDF2 import PdfReader
from http import HTTPStatus
from flask import current_app, request, make_response
from flask_restx import Namespace, Resource, fields
from formsflow_api_utils.utils import (
    cors_preflight,
    auth,
    profiletime
)
from formsflow_api.schemas import DocumentsListSchema
from formsflow_api_utils.utils.user_context import UserContext, user_context
from formsflow_api_utils.exceptions import BusinessException
from formsflow_api.exceptions import EurotrustException
from formsflow_api.services import FormioServiceExtended, DocumentsService, KeycloakAdminAPIService
from formsflow_api.services.external import EurotrustIntegrationsService
from formsflow_api.models import DocumentStatus

API = Namespace("Documents", description="Temporary documents EP")

document = API.model(
    "Document",
    {
        "id"
    }
)

VALID_STATUSES = [
    "unsigned",
    "signing",
    "signed",
    "expired",
    "rejected"
]

VALID_TYPES = [
    "eAuth"
    "eIdentity"
    "OKEP"
    "eDelivery"
]

@cors_preflight("GET,OPTIONS")
@API.route("", methods=["GET", "OPTIONS"])
class DocumentResource(Resource):
    """Resource for managing documents."""

    @classmethod
    @auth.require
    @profiletime
    @user_context
    @API.doc(
        params={
            "status": {
                "in": "query",
                "description": "Status filter. Valid statuses are: " + ",".join(VALID_STATUSES),
                "default": None
            },
            "createdAfter": {
                "in": "query",
                "description": "Created after filter: ",
                "default": None
            },
            "cursor": {
                "in": "query",
                "description": "Current page",
                "default": None
            },
        }
    )
    def get(cls, **kwargs):
        ### 1. Get user data
        current_app.logger.debug("1. Get user data")
        user: UserContext = kwargs["user"]
        is_verified = getattr(user, 'is_verified', False)
        current_app.logger.debug(f"Is verified - {is_verified}")
        person_identifier = user.token_info["personIdentifier"]
        tenant_key = user.tenant_key

        current_app.logger.debug("2. Check person identifier validity")
        match = re.findall(r"PNOBG-(\d{10})$", person_identifier)
        if not match:
            current_app.logger.debug(BusinessException)
            raise BusinessException(
                "Invalid Personal Identifier bound to user.", HTTPStatus.BAD_REQUEST
            )

        current_app.logger.debug("3. See if the user is verified")
        if not is_verified:
            current_app.logger.debug("4. Try and get an identity_request if exists")
            person_identifier = match[0]
            document_client = DocumentsService()
            identity_request = document_client.get_identity_request(
                person_identifier=person_identifier,
                tenant_key=tenant_key
            )

            if not identity_request:
                identity_request = document_client.create_identity_request(
                    person_identifier=person_identifier,
                    tenant_key=tenant_key
                )

            current_app.logger.debug("5. Spoof response")
            return_dict = {
                "documents": [
                    {
                        "created": identity_request.created.isoformat(),
                        "modified": identity_request.modified.isoformat(),
                        "expired": None,
                        "validUntill": identity_request.valid_untill.isoformat(),
                        "signed": None,
                        "rejected": None,
                        "applicationId": None,
                        "evrotrustThreadId": identity_request.thread_id,
                        "evrotrustTransactionId": identity_request.transaction_id,
                        "fileName": None,
                        "status": "pending",
                        "formioId": None,
                        "referenceNumber": None,
                        "fileUrl": None,
                    }
                ],
                "pagination": {
                    "cursor": None
                }
            }

            return (return_dict, HTTPStatus.OK)

        ## Paginate data
        current_app.logger.debug("4. User is verified, get cursor parameters")
        args = DocumentsListSchema().load(request.args) or {}
        new_cursor = args.get('cursor', None)
        if new_cursor:
            pagination_values = base64.urlsafe_b64decode(new_cursor)
            pagination_values = pagination_values.decode()
            pagination_values = json.loads(pagination_values)
        else:
            pagination_values = {
                "skip": 0,
                "limit": args.get("limit", 5)
            }

        skip = pagination_values.get('skip')
        limit = pagination_values.get("limit")

        current_app.logger.debug("5. Init formio client")
        formio_client = FormioServiceExtended()
        formio_token = formio_client.get_formio_access_token()
        file_formio_path = user.tenant_key + "-" + current_app.config.get('FORMIO_FILE_RESOURCE_PATH')

        status = args.get("status", [])
        if status:
            status = status.split(",")

        current_app.logger.debug("6. Get submissions, getting one more to see if there are further pages")
        related_file_submissions, formio_status = formio_client.get_submissions(
            file_formio_path,
            formio_token=formio_token,
            person_identifier=person_identifier,
            select=['_id', 'data', 'created', 'modified'],
            status=status,
            skip=skip,
            limit=limit + 1, ### Get one more to check if we have more pages
            created_after=args.get("created_after")
        )

        current_app.logger.debug("7. Encode new cursor")
        ### If the returned submissions (limit + 1) are more than the limit, there is another page
        if len(related_file_submissions) > limit:
            skip += limit
            next_cursor = base64.urlsafe_b64encode(
                    json.dumps(
                        {"skip": skip, "limit": limit}
                    ).encode()
                ).decode()
        else:
            next_cursor=None

        current_app.logger.debug("8. Format data")
        returned_submissions = []
        current_index = 0

        api_base = current_app.config.get("FORMSFLOW_API_URL")

        for submission in related_file_submissions:
            if current_index == limit:
                break
            current_index += 1
            submission_data = submission.get("data", {})
            file_list = submission_data.get("file", [])
            if not file_list:
                file_data = {}
            else:
                file_data = file_list[0]

            formio_id = submission.get("_id")
            returned_submissions.append(
                {
                    "created": submission.get("created"),
                    "modified": submission.get("modified"),
                    "expired": submission_data.get("expired"),
                    "validUntill": submission_data.get("validUntill"),
                    "signed": submission_data.get("signed"),
                    "rejected": submission_data.get("rejected"),
                    "applicationId": submission_data.get("applicationId"),
                    "evrotrustThreadId": submission_data.get("evrotrustThreadId"),
                    "evrotrustTransactionId": submission_data.get("evrotrustTransactionId"),
                    "fileName": file_data.get("name"),
                    "status": submission_data.get("status"),
                    "formioId": submission.get("_id"),

                    ### To add
                    "referenceNumber": submission_data.get("referenceId", "TODO"),
                    "fileUrl": f"{api_base}/documents/{formio_id}/serve",
                }
            )

        response, formio_status = {
                "documents": returned_submissions,
                "pagination": {
                    "cursor": next_cursor
                }
            }, formio_status
        return response, formio_status

@cors_preflight("GET,OPTIONS")
@API.route("/<string:submission_formio_id>/serve", methods=["GET", "OPTIONS"])
class DocumentServeBase64(Resource):

    @staticmethod
    @auth.require
    @profiletime
    @user_context
    @API.response(200, "OK:- Successful request.")
    @API.response( 400, "BAD_REQUEST:- Invalid request.")
    @API.response( 404, "NOT_FOUND:- No document found to check status.")
    def get(submission_formio_id: str, **kwargs):
        try:
            user: UserContext = kwargs["user"]
            person_identifier = user.token_info["personIdentifier"]
            formio_client = FormioServiceExtended()

            form_path = user.tenant_key + "-" + current_app.config.get('FORMIO_FILE_RESOURCE_PATH')
            formio_token = formio_client.get_formio_access_token()
            file_form_id = formio_client.fetch_form_id_by_path(form_path=form_path, formio_token=formio_token)

            response = formio_client.debug_get_submission(
                data={
                    "form_id": file_form_id,
                    "sub_id": submission_formio_id
                },
                formio_token=formio_token
            )

            submission_data = response.get("data")
            file_data = submission_data.get("file", [])[-1]
            user_id = submission_data.get("userId")

            if (person_identifier != user_id) or (not file_data):
                return {
                    "type": "Not found",
                    "message": f"No document with supplied formioId {submission_formio_id} found"
                }, HTTPStatus.NOT_FOUND

            file = file_data['url']
            file = re.split(";|,", file)

            ### If the string is as following - data:application/pdf;base64,JVBERi0xLjQKJdPr6eEKMSAwIG9ia...
            ### The above split will make a file corresponding to:
            ### 0 - data:application/pdf - file data type
            ### 1 - base64 - encoding type
            ### 2 - JVBERi0xLjQKJdPr6eEKMSAw... -

            image_binary = base64.b64decode(file[2])
            response = make_response(image_binary)
            current_app.logger.debug(file_data['name'])
            response.headers.set('Content-Type', file_data['type'])
            response.headers.set(
                'Content-Disposition', 'inline', filename=file_data['name'].encode("utf-8"))
            return response

        except BusinessException as err:
            current_app.logger.error(err.error)
            current_app.logger.error(err.status_code)
            if err.status_code == 400:
                response, status = {
                    "type": "Not found",
                    "message": f"No document with supplied formioId {submission_formio_id} found",
                }, HTTPStatus.BAD_REQUEST
            else:
                response, status = {
                    "type": "Bad request error",
                    "message": "Internal server error",
                }, HTTPStatus.BAD_REQUEST

            return response, status

@cors_preflight("GET,OPTIONS")
@API.route("/<string:submission_formio_id>/status", methods=["GET", "OPTIONS"])
class DocumentCheckStatus(Resource):

    @staticmethod
    @auth.require
    @profiletime
    @user_context
    @API.response(200, "OK:- Successful request.")
    @API.response( 400, "BAD_REQUEST:- Invalid request.")
    @API.response( 404, "NOT_FOUND:- No document found to check status.")
    def get(submission_formio_id: str, **kwargs):
        try:
            ### 0. Get user data
            user: UserContext = kwargs["user"]
            person_identifier = user.token_info["personIdentifier"]
            current_app.logger.debug(f"Person Identifier - {person_identifier}")

            match = re.findall(r"PNOBG-(\d{10})$", person_identifier)
            if not match:
                raise BusinessException(
                    "Invalid Personal Identifier bound to user.", HTTPStatus.BAD_REQUEST
                )

            ### 1. Get document resource by submission_formio_id and pnobg
            formio_client = FormioServiceExtended()
            formio_file_resource_path = user.tenant_key + "-" + current_app.config.get('FORMIO_FILE_RESOURCE_PATH')
            formio_token = formio_client.get_formio_access_token()
            generated_files_formio_form_id = formio_client.fetch_form_id_by_path(
                form_path=formio_file_resource_path,
                formio_token=formio_token
            )

            generated_file_resource = formio_client.debug_get_submission(
                data={
                    'form_id': generated_files_formio_form_id,
                    'sub_id': submission_formio_id
                },
                formio_token=formio_token
            )

            ### 2. Check document status in eurotrust
            generated_file_resource_data = generated_file_resource.get("data")
            generated_file_resource_transaction_id = generated_file_resource_data.get("evrotrustTransactionId")

            eurotrust_client = EurotrustIntegrationsService()
            response = eurotrust_client.check_document_status(transaction_id=generated_file_resource_transaction_id)
            eurotrust_status = response["status"]
            status = DocumentStatus.query.filter_by(eurotrust_status=eurotrust_status).first()

            ### 3. Update status
            if not status:
                raise BusinessException(
                    f"No status found {eurotrust_status} in our system",
                    HTTPStatus.NOT_FOUND
                )

            document_service_client = DocumentsService()
            update_response = document_service_client.update_document_status_in_formio(
                formio_id=submission_formio_id,
                status=status
            )

            return (update_response, HTTPStatus.OK)
        except BusinessException as err:
            response, status = {
                "type": "Bad request error",
                "message": (err),
            }, HTTPStatus.BAD_REQUEST

            current_app.logger.warning(response)
            current_app.logger.warning(err)

            return response, status

@cors_preflight("GET,OPTIONS")
@API.route("/authenticate/<string:transaction_id>", methods=["GET", "OPTIONS"])
class AuthenticateResource(Resource):

    @staticmethod
    @auth.require
    @profiletime
    @user_context
    @API.response(200, "OK:- Successful request.")
    @API.response( 400, "BAD_REQUEST:- Invalid request.")
    @API.response( 404, "NOT_FOUND:- No document found to check status.")
    def get(transaction_id: str,**kwargs):
        try:
            user: UserContext = kwargs["user"]
            user_keycloak_id = user.token_info["sub"]

            eurotrust_client = EurotrustIntegrationsService()
            response = eurotrust_client.check_document_status(transaction_id=transaction_id)
            current_app.logger.debug(response.get("status"))
            status = DocumentStatus.query.filter_by(eurotrust_status=response.get("status")).first()
            current_app.logger.debug(status)

            if not status:
                return {
                    "type": "Invalid Status",
                    "message": "Invalid status returned from EuroTrust",
                }, HTTPStatus.BAD_REQUEST

            if status.formio_status != "signed":
                return {
                    "type": "Not signed",
                    "message": "Status for transaction is not signed",
                }, HTTPStatus.BAD_REQUEST

            file_list = eurotrust_client.get_signed_file(transaction_id=transaction_id)

            signed_file_dict = file_list[0]
            signed_file_content = signed_file_dict["content"]
            base64_bytes = BytesIO(base64.b64decode(signed_file_content))

            reader = PdfReader(base64_bytes)
            meta = reader.metadata
            keywords_string = meta.get("/Keywords")

            base64_bytes.close()
            if keywords_string and user_keycloak_id:
                keycloak_client = KeycloakAdminAPIService()
                initial_client = keycloak_client.get_request(url_path=f"users/{user_keycloak_id}")

                xml_string = base64.b64decode(keywords_string)
                xml_tree = ElementTree.ElementTree(ElementTree.fromstring(xml_string))
                root = xml_tree.getroot()

                attributes = initial_client.get("attributes")
                attributes["isVerified"] = True

                for child in root:
                    if not child.tag.startswith("pic"):
                        attributes["evroTrust" + child.tag.capitalize()] = child.text

                response = keycloak_client.update_request(
                    url_path=f"users/{user_keycloak_id}",
                    data={
                        "attributes": attributes
                    }
                )

            return (
                {
                    "status": status.formio_status
                },
                HTTPStatus.OK
            )

        except EurotrustException as err:
            current_app.logger.warning(err)
            return err.error, err.status_code


from fastapi.testclient import TestClient
from sqlalchemy.orm import Session
from sqlalchemy import and_
from app import models

def test_desire_create(client: TestClient, db: Session, seller_bearer_token: str):
    cfa_image_id = client.post("/cfa-image/create", json={
        "count": 100,
        "description": "test cfa-image",
        "title": "test cfa-image"
    }, headers={"Authorization": f"Bearer {seller_bearer_token}"}).json()["id"]

    response_desire_create = client.post("/desire/create", json={
        "cfa_image_id": cfa_image_id,
        "count": 100,
        "price": 100,
    })

    assert response_desire_create.is_success

def test_desire_sell(client: TestClient, seller_bearer_token: str):
    cfa_image_id = client.post("/cfa-image/create", json={
        "count": 100,
        "description": "test cfa-image",
        "title": "test cfa-image"
    }, headers={"Authorization": f"Bearer {seller_bearer_token}"}).json()["id"]

    response_desire_create = client.post("/desire/create", json={
        "cfa_image_id": cfa_image_id,
        "count": 100,
        "price": 100,
    }).json()

    client.post("/user/deposit", json={"value": 1000000})
    pre_user = client.get("user/profile").json()

    response = client.post(f"/desire/sell/{response_desire_create['id']}",
                           headers={"Authorization": f"Bearer {seller_bearer_token}"},
                           json={"count": 100},
                           )
    assert response.is_success
    after_user = client.get("user/profile").json()
    assert pre_user["balance"] - 100 * 100 == after_user["balance"]

def test_desire_cancel(client: TestClient, seller_bearer_token):
    cfa_image_id = client.post("/cfa-image/create", json={
        "count": 100,
        "description": "test cfa-image",
        "title": "test cfa-image"
    }).json()["id"]

    response_desire_create = client.post("/desire/create", json={
        "cfa_image_id": cfa_image_id,
        "count": 100,
        "price": 100,
    }).json()
    user_desires = client.get(f"/desire/list/{cfa_image_id}").json()
    desire_id = user_desires[0]["id"]

    # cancel by another person
    response = client.post(f"/desire/cancel/{desire_id}", headers={"Authorization": f"Bearer {seller_bearer_token}"})
    assert not response.is_success

    # cancel by another person
    response = client.post(f"/desire/cancel/{desire_id}")
    assert response.is_success

    assert len(user_desires) - 1 == len(client.get(f"/desire/list/{cfa_image_id}").json())

def test_desire_list(client: TestClient, user_id: int):
    cfa_image_id = client.post("/cfa-image/create", json={
        "count": 111,
        "description": "test cfa-image",
        "title": "test cfa-image"
    }).json()["id"]

    response_desire_create = client.post("/desire/create", json={
        "cfa_image_id": cfa_image_id,
        "count": 100,
        "price": 100,
    }).json()

    response_desire_create = client.post("/desire/create", json={
        "cfa_image_id": cfa_image_id,
        "count": 10,
        "price": 10,
    }).json()

    response_desire_create = client.post("/desire/create", json={
        "cfa_image_id": cfa_image_id,
        "count": 1,
        "price": 1,
    }).json()

    desire_list_response = client.get(f"desire/list/{cfa_image_id}")

    assert desire_list_response.is_success
    assert len(desire_list_response.json()) == 3


from django.urls import path
from App1 import views
from App1.views import curso

urlpatterns = [
    path('curso/', curso),
    path('', views.inicio, name="Inicio"),
    path('cursos', views.cursos, name="Cursos"),
    path('profesores', views.profesores, name="Profesores"),
    path('estudiantes', views.estudiantes, name="Estudiantes"),
    path('entregables', views.entregables, name="Entregables"),

]


import os
import sys
from typing import Optional, Tuple

def glibc_version_string() -> Optional[str]:
    "Returns glibc version string, or None if not using glibc."
    return glibc_version_string_confstr() or glibc_version_string_ctypes()

def glibc_version_string_confstr() -> Optional[str]:
    "Primary implementation of glibc_version_string using os.confstr."
    # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely
    # to be broken or missing. This strategy is used in the standard library
    # platform module:
    # https://github.com/python/cpython/blob/fcf1d003bf4f0100c9d0921ff3d70e1127ca1b71/Lib/platform.py#L175-L183
    if sys.platform == "win32":
        return None
    try:
        gnu_libc_version = os.confstr("CS_GNU_LIBC_VERSION")
        if gnu_libc_version is None:
            return None
        # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17":
        _, version = gnu_libc_version.split()
    except (AttributeError, OSError, ValueError):
        # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)...
        return None
    return version

def glibc_version_string_ctypes() -> Optional[str]:
    "Fallback implementation of glibc_version_string using ctypes."

    try:
        import ctypes
    except ImportError:
        return None

    # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
    # manpage says, "If filename is NULL, then the returned handle is for the
    # main program". This way we can let the linker do the work to figure out
    # which libc our process is actually using.
    process_namespace = ctypes.CDLL(None)
    try:
        gnu_get_libc_version = process_namespace.gnu_get_libc_version
    except AttributeError:
        # Symbol doesn't exist -> therefore, we are not linked to
        # glibc.
        return None

    # Call gnu_get_libc_version, which returns a string like "2.5"
    gnu_get_libc_version.restype = ctypes.c_char_p
    version_str = gnu_get_libc_version()
    # py2 / py3 compatibility:
    if not isinstance(version_str, str):
        version_str = version_str.decode("ascii")

    return version_str

# platform.libc_ver regularly returns completely nonsensical glibc
# versions. E.g. on my computer, platform says:
#
#   ~$ python2.7 -c 'import platform; print(platform.libc_ver())'
#   ('glibc', '2.7')
#   ~$ python3.5 -c 'import platform; print(platform.libc_ver())'
#   ('glibc', '2.9')
#
# But the truth is:
#
#   ~$ ldd --version
#   ldd (Debian GLIBC 2.22-11) 2.22
#
# This is unfortunate, because it means that the linehaul data on libc
# versions that was generated by pip 8.1.2 and earlier is useless and
# misleading. Solution: instead of using platform, use our code that actually
# works.
def libc_ver() -> Tuple[str, str]:
    """Try to determine the glibc version

    Returns a tuple of strings (lib, version) which default to empty strings
    in case the lookup fails.
    """
    glibc_version = glibc_version_string()
    if glibc_version is None:
        return ("", "")
    else:
        return ("glibc", glibc_version)