from abc import ABC
import numpy as np

import matplotlib as mpl
import matplotlib.pyplot as plt

from scipy.constants import c
from scipy.optimize import curve_fit
from scipy.ndimage import gaussian_filter1d

import os
from glob import glob

from astropy.table import Table, join, vstack, hstack, unique

from desispec.io import read_spectra
from desispec.spectra import Spectra
from desispec.coaddition import coadd_cameras

import redrock.templates

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore", category=RuntimeWarning)
#warnings.filterwarnings("ignore", category=OptimizeWarning)
mpl.rc('font', size=14)

class SpectralLine(ABC):

    def __init__(self, name, wavelength):

        self.name = name
        self.line = wavelength

    def get_linewidth(self, wave, flux):
        pass
#Just a naive gaussian definition for later fitting
def gaus(x, sigma, line, A, b):
    flux = A*np.exp(-(x - line)**2/(2*sigma)**2) + b
    return flux

class SpectralLine_pEW(SpectralLine):

    def __init__(self, name, wavelength, cont_blue=None, cont_red=None):
        """Initialize a spectral line used for a pEW calculation.

        Parameters
        ----------
        line : str
            Name of line.
        wavelength : float
            restframe wavelength of line.
        cont_blue : list of two floats, or None
            Lower and upper wavelength for computing continuum on 'blue' end of line.
        cont_red : list of two floats, or None
            Lower and upper wavelength for computing continuum on 'red' end of line.
        """
        super().__init__(name, wavelength)

        if cont_blue is not None:
            self.cont_blue = cont_blue
        else:
            self.cont_blue = [self.line - 205., self.line - 195.]

        if cont_red is not None:
            self.cont_red = cont_red
        else:
            self.cont_red = [self.line + 195., self.line + 205.]

        self.vshift = None
        self.roi_wave = None
        self.roi_flux = None
        self.roi_flux_scaled = None
        self.roi_continuum = None

    def get_roi(self, wave, flux, dminwave=200., dmaxwave=200.):
        """Produce a region of interest for estimating the line pEW.

        Parameters
        ----------
        wave : ndarray
            Input wavelength from a spectrum.
        flux : ndarray
            Input flux from a spectrum.
        dminwave : float
            Distance below the line center (in Angstrom) for computing the ROI.
        dmaxwave : float
            Distance above the line center (in Angstrom) for computing the ROI.

        Returns
        -------
        vshift : ndarray
            Array of velocities in the ROI w.r.t. to the line rest frame.
        roi_wave : ndarray
            Selected wavelengths in the ROI.
        roi_flux : ndarray
            Selected flux values in the ROI.
        """
        if wave[-1] < self.line - dminwave or wave[1] > self.line + dmaxwave:
            raise ValueError('Analysis region not covered for {} line'.format(self.name))

        # Get velocity shift
        vshift = c * (wave / self.line - 1.)

        # Select the wavelengths in the region of interest around the line.
        select = (wave > self.line - dminwave) & (wave < self.line + dmaxwave)

        return vshift[select], wave[select], flux[select]

    def estimate_continuum(self, roi_wave, roi_flux):
        """Perform a linear fit of the continuum using 'blue' and 'red' sidebands of the spectra line.

        Parameters
        ----------
        roi_wave : ndarray
            Wavelengths in the region of interest of this spectral line.
        roi_flux : ndarray
            Fluxes in the region of interest of the spectral line.

        Returns
        -------
        a : float
            Intercept of the linear fit to the continuum.
        b : float
            Slope of the linear fit to the continuum.
        continuum : ndarray
            Computed continuum under the spectral line using a linear model.
        scaled_roi_flux :
            Flux in the ROI, scaled by the continuum to remove its slope.
        """
        blue_min, blue_max = self.cont_blue
        blue_avg = 0.5*(blue_min + blue_max)
        select_blue = (roi_wave > blue_min) & (roi_wave < blue_max)
        flux_blue = roi_flux[select_blue]

        red_min, red_max = self.cont_red
        red_avg = 0.5*(red_min + red_max)
        select_red = (roi_wave > red_min) & (roi_wave < red_max)
        flux_red = roi_flux[select_red]

        # Perform a linear fit to the mean flux vs mean wavelength in the blue and red sidebands of the line.
        # a = intercept, b = slope.
        b = (np.nanmean(flux_red) - np.nanmean(flux_blue)) / (red_avg - blue_avg)
        a = np.nanmean(flux_blue) - (b * blue_avg)

        # Calculate the continuum using the linear fit.
        continuum = a + b*roi_wave

        # Scale the flux by the linear continuum.
        scaled_roi_flux = roi_flux / continuum
        return a, b, continuum, scaled_roi_flux

    def estimate_peqw(self, roi_wave, scaled_roi_flux):
        """Compute the pseudo-equivalent width of the spectral line.

        Parameters
        ----------
        roi_wave : ndarray
            Wavelength in the fit region of interest of the line.
        scaled_roi_flux : ndarray
            Flux in the ROI scaled by the continuum to remove its slope.

        Returns
        -------
        """
        continuum = np.ones_like(scaled_roi_flux)
        _sum = np.sum((scaled_roi_flux - continuum) / continuum)
        _delta = np.diff(roi_wave)[0]
        return _sum * _delta

    def get_linewidth(self, wave, flux):
        """Return the line width (pseudo-equivalent width).

        Parameters
        ----------
        wave : ndarray
            Input wavelength from a spectrum.
        flux : ndarray
            Input flux from a spectrum.

        Returns
        -------
        peqw : float
            Pseudo-equivalent width of the spectral line.
        """
        vs, roi_wave, roi_flux = self.get_roi(wave, flux,self.line - np.nanmean(self.cont_blue),\
                                              np.nanmean(self.cont_red)-self.line)
        a, b, continuum, scaled_roi_flux = self.estimate_continuum(roi_wave, roi_flux)
        peqw = self.estimate_peqw(roi_wave, scaled_roi_flux)

        # Store the details of the continuum fit for debugging.
        self.vshift = vs
        self.roi_wave = roi_wave
        self.roi_flux = roi_flux
        self.roi_flux_scaled = scaled_roi_flux
        self.roi_continuum = continuum

        return peqw

    def __str__(self):
        s = '{}: {}\nBlue continuum fit: {} - {}\nRed continuum fit:  {} - {}'.format(
            self.name, self.line,
            self.cont_blue[0], self.cont_blue[1],
            self.cont_red[0], self.cont_red[1])
        return s

def is_TDE(Ha,HB,HeII,OIII,NIII, flux):
    '''
    Ha,HB, HeII, and OIII should be pEW for each of those lines respectively.
    '''
    TDE_score = 0
    blue_end = np.nanmean(flux[1000:3000])
    red_end = np.nanmean(flux[5000:7000])
    mean_flux = np.nanmean(flux)
    try:
        if OIII/Ha > 0.1:
            TDE_score -= 1.1 #if OIII is present, will be marked by an index with a noninteger part
    except ZeroDivisionError:
        pass
    if Ha > 4: #Just looks for generally wide Ha lines
        TDE_score += 1
    if HB > 4:
        TDE_score += 1
    try:
        if HeII/Ha >0.75: #looks for comparable He and Ha lines
            TDE_score += 1
    except ZeroDivisionError:
        pass
    if NIII > 4: #looks for NIII sometimes seen in TDE-bowen
        TDE_score += 1
    if blue_end/red_end >= 1.25:
        TDE_score += 1
    if mean_flux < 1:
        TDE_score -= 1.1

    return TDE_score

def Combine_multifilt(wave,flux, mask = False):
        difwave_single = []
        difflux_single = []
        difmask_single = []
        for band in wave:
            difwave_single += list(wave[band])
            difflux_single += list(flux[band])
            if mask:
                difmask_single += list(mask[band])

        if mask:
            diftable = Table([difwave_single, difflux_single,difmask_single], names = ('wave','flux', 'mask'))
        else:
            diftable = Table([difwave_single, difflux_single], names = ('wave','flux'))
        diftable.sort('wave')
        difwave_single = list(diftable['wave'])
        difflux_single = list(diftable['flux'])
        if mask:
            difmask_single = list(diftable['mask'])
            return difwave_single,difflux_single,difmask_single
        else:
            return difwave_single,difflux_single

def TDE_Check(wave,flux, redshift, scale = 3.,cont_width = 10,multifilt = False, spectype = 'GALAXY', mask = False):
    '''
    Creates a scaled flux (assumed to be roughly gaussian) for each line of interest. Then fits this scaled flux to a
    Gaussian, pulling the $\sigma$. Recalculates pEW using an ROI of the line $\pm$ scale*sigma. Appends this pEW to
    a Dictionary (lines), each line should have a list in format [wavelength, pEW]. pEW used in logic to determine whether
    or not it's a TDE.
    Parameters:
    wave, flux: wavelength and flux of spectrum
    scale: # of standard deviations to fit continuum
    cont_width: width of region to sample cont_blue and cont_red, default 5 grabs area 2.5 angstroms on either side of
        bound determined by scale*sigma
    '''
    if spectype == 'GALAXY':

        if multifilt:
            if mask:
                wave,flux, mask = Combine_multifilt(wave, flux,mask)
            else:
                wave,flux= Combine_multifilt(wave, flux)
        wave = wave/(1+redshift)
        flux = gaussian_filter1d(flux, sigma = 3)
        half_cont = cont_width/2
        lines = {'Halpha':[6562.79],'Hbeta':[4861.4], 'HeII4686':[4686],'OIII':[5007],'NIII':[4100]}
        for line in lines:
            pass1 = SpectralLine_pEW(line,lines[line][0])
            try:
                peqw_pass1 = pass1.get_linewidth(wave, flux)
                #plt.plot(pass1.roi_wave, pass1.roi_flux)
                #plt.show()
                #print(line)
                #print(peqw_pass1)

                try:
                    popt = curve_fit(gaus, pass1.roi_wave, pass1.roi_flux_scaled, p0 = [10.,pass1.line,max(pass1.roi_flux_scaled),1],check_finite = False)[0]
                    std = popt[0]
                    #print(std)
                    if scale*std <= half_cont: #Fit failed, line probably not present enough for measurement.
                        peqw_specline = 0
                        #print('Failed')

                    else:
                        cont_blue=[pass1.line - scale*std-half_cont, pass1.line - scale*std+half_cont]
                        cont_red=[pass1.line + scale*std-half_cont, pass1.line + scale*std+half_cont]
                        specline = SpectralLine_pEW(line, lines[line][0], cont_blue=cont_blue, \
                                    cont_red=cont_red)
                        if mask:
                            roistart_ind = wave.index(specline.roi_wave[0])
                            roiend_ind = wave.index(specline.roi_wave[-1]) + 1
                            if 1 in mask[roistart_ind:roiend_ind]:
                                peqw_specline = 0

                            else:
                                try:
                                    peqw_specline = specline.get_linewidth(wave, flux)
                                    #print(cont_blue,cont_red)
                                    #print(peqw_specline)
                                    #plt.plot(specline.roi_wave, specline.roi_flux)
                                    #plt.show()
                                except IndexError:
                                    peqw_specline = 0
                #print('###########')
                except RuntimeError:
                    peqw_specline = 0
            except:
                peqw_specline = 0
            lines[line].append(peqw_specline)
        return is_TDE(lines['Halpha'][-1],lines['Hbeta'][-1],lines['HeII4686'][-1],lines['OIII'][-1],\
                      lines['NIII'][-1], flux)
    else:
        return 0


import os
import hashlib
import base64
import urllib
import string
import numbers
import threading
import errno
import fnmatch
import json

def makedirs_p(dirname, mode = None, reason = None):
    """
    Create a directory tree, ignoring an error if it already exists

    :param str pathname: directory tree to crate
    :param int mode: mode set the directory to
    """
    try:
        os.makedirs(dirname)
        # yes, this is a race condition--but setting the umask so
        # os.makedirs() gets the right mode would interfere with other
        # threads and processes.
        if mode:
            os.chmod(dirname, mode)
    except OSError as e:
        if not os.path.isdir(dirname):
            raise RuntimeError("%s: path for %s is not a directory: %s"
                               % (dirname, reason, e))
        if not os.access(dirname, os.W_OK):
            raise RuntimeError("%s: path for %s does not allow writes: %s"
                               % (dirname, reason, e))

def flat_keys_to_dict(d):
    """
    Given a dictionary of flat keys, convert it to a nested dictionary

    Similar to :func:`flat_slist_to_dict`, differing in the
    keys/values being in a dictionary.

    A key/value:

    >>> d["a.b.c"] = 34

    means:

    >>> d['a']['b']['c'] = 34

    Key in the input dictonary are processed in alphabetical order
    (thus, key a.a is processed before a.b.c); later keys override
    earlier keys:

    >>> d['a.a'] = 'aa'
    >>> d['a.a.a'] = 'aaa'
    >>> d['a.a.b'] = 'aab'

    will result in:

    >>> d['a']['a'] = { 'a': 'aaa', 'b': 'aab' }

    The

    >>> d['a.a'] = 'aa'

    gets overriden by the other settings

    :param dict d: dictionary of keys/values
    :returns dict: (nested) dictionary
    """
    tr = {}

    for key in sorted(d.keys()):
        _key_rep(tr, key, key, d[key])

    return tr

def rm_f(filename):
    """
    Remove a file (not a directory) unconditionally, ignore errors if
    it does not exist.
    """
    try:
        os.unlink(filename)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def mkid(something, l = 10):
    """
    Generate a 10 character base32 ID out of an iterable object

    :param something: anything from which an id has to be generate
      (anything iterable)
    """
    if isinstance(something, str):
        h = hashlib.sha512(something.encode('utf-8'))
    else:
        h = hashlib.sha512(something)
    return base64.b32encode(h.digest())[:l].lower().decode('utf-8', 'ignore')

class fsdb_c(object):
    """
    This is a very simple key/value flat database

    - sets are atomic and forcefully remove existing values
    - values are just strings
    - value are limited in size to 1K
    - if a field does not exist, its value is *None*

    The key space is flat, but with a convention of periods dividing
    fields, so that:

      l['a.b.c'] = 3

    is the equivalent to:

      l['a']['b']['c'] = 3

    it also makes it way faster and easier to filter for fields.

    This will be used to store data for each target; for implemntation
    examples, look at :class:`commonl.fsdb_symlink_c`.
    """
    class exception(Exception):
        pass

    def keys(self, pattern = None):
        """
        List the fields/keys available in the database

        :param str pattern: (optional) pattern against the key names
          must match, in the style of :mod:`fnmatch`. By default, all
          keys are listed.

        :returns list: list of keys
        """
        raise NotImplementedError

    def get_as_slist(self, *patterns):
        """
        Return a sorted list of tuples *(KEY, VALUE)*\s available in the
        database.

        :param list(str) patterns: (optional) list of patterns of fields
          we must list in the style of :mod:`fnmatch`. By default, all
          keys are listed.

        :returns list(str, str): list of *(KEY, VALUE)* sorted by
          *KEY* (so *a.b.c*, representing *['a']['b']['c']* goes
          after *a.b*, representing *['a']['b']*).
        """
        raise NotImplementedError

    def get_as_dict(self, *patterns):
        """
        Return a dictionary of *KEY/VALUE*\s available in the
        database.

        :param str pattern: (optional) pattern against the key names
          must match, in the style of :mod:`fnmatch`. By default, all
          keys are listed.

        :returns dict: keys and values in dictionary form
        """
        raise NotImplementedError

    def set(self, key, value, force = True):
        """
        Set a value for a key in the database unless *key* already exists

        :param str key: name of the key to set

        :param str value: value to store; *None* to remove the field;
          only *string*, *integer*, *float* and *boolean* types

        :parm bool force: (optional; default *True*) if *key* exists,
          force the new value

        :return bool: *True* if the new value was set correctly;
          *False* if *key* already exists and *force* is *False*.
        """
        assert isinstance(value, (NoneType, str, int, float, bool)), \
            f"value must be None, str, int, float, bool; got {type(value)}"

    def get(self, key, default = None):
        """
        Return the value stored for a given key

        :param str key: name of the key to retrieve

        :param str default: (optional) value to return if *key* is not
          set; defaults to *None*.

        :returns str: value associated to *key* if *key* exists;
          otherwise *default*.
        """
        raise NotImplementedError

    @staticmethod
    def create(cache_dir):
        """
        Create with the right type for the host OS

        Same params as :class:`fsdb_symlink_c`

        Note there are catchas for atomicity; read the docs for:

        - :class:`fsdb_symlink_c`
        - :class:`fsdb_file_c`
        """
        if sys.platform in ( 'linux', 'macos' ):
            return fsdb_symlink_c(cache_dir)
        else:
            return fsdb_file_c(cache_dir)

class fsdb_symlink_c(fsdb_c):
    """
    This implements a database by storing data on the destination
    argument of a Unix symbolic link

    Creating a symlink, takes only one atomic system call, which fails
    if the link already exists. Same to read it. Thus, for small
    values, it is very efficient.
    """
    class invalid_e(fsdb_c.exception):
        pass

    def __init__(self, dirname, use_uuid = None, concept = "directory"):
        """
        Initialize the database to be saved in the give location
        directory

        :param str location: Directory where the database will be kept
        """
        if not os.path.isdir(dirname):
            raise self.invalid_e("%s: invalid %s"
                                 % (os.path.basename(dirname), concept))
        if not os.access(dirname, os.R_OK | os.W_OK | os.X_OK):
            raise self.invalid_e("%s: cannot access %s"
                                 % (os.path.basename(dirname), concept))

        if use_uuid == None:
            self.uuid = mkid(str(id(self)) + str(os.getpid()))
        else:
            self.uuid = use_uuid

        self.location = dirname

    def _raw_valid(self, location):
        return os.path.islink(location)

    def _raw_read(self, location):
        return os.readlink(location)

    def _raw_write(self, location, value):
        os.symlink(value, location)

    def _raw_unlink(self, location):
        os.unlink(location)

    def _raw_rename(self, location_new, location):
        os.replace(location_new, location)

    @staticmethod
    def _raw_stat(location):
        return os.lstat(location)

    def keys(self, pattern = None):
        l = []
        for _rootname, _dirnames, filenames_raw in os.walk(self.location):
            filenames = []
            for filename_raw in filenames_raw:
                # need to filter with the unquoted name...
                filename = urllib.parse.unquote(filename_raw)
                if pattern == None or fnmatch.fnmatch(filename, pattern):
                    if self._raw_valid(os.path.join(self.location, filename_raw)):
                        l.append(filename)
        return l

    def get_as_slist(self, *patterns):
        fl = []
        for _rootname, _dirnames, filenames_raw in os.walk(self.location):
            filenames = {}
            for filename in filenames_raw:
                filenames[urllib.parse.unquote(filename)] = filename
            if patterns:	# that means no args given
                use = {}
                for filename, filename_raw in filenames.items():
                    if field_needed(filename, patterns):
                        use[filename] = filename_raw
            else:
                use = filenames
            for filename, filename_raw in use.items():
                if self._raw_valid(os.path.join(self.location, filename_raw)):
                    bisect.insort(fl, ( filename, self._get_raw(filename_raw) ))
        return fl

    def get_as_dict(self, *patterns):
        d = {}
        for _rootname, _dirnames, filenames_raw in os.walk(self.location):
            filenames = {}
            for filename in filenames_raw:
                filenames[urllib.parse.unquote(filename)] = filename
            if patterns:	# that means no args given
                use = {}
                for filename, filename_raw in filenames.items():
                    if field_needed(filename, patterns):
                        use[filename] = filename_raw
            else:
                use = filenames
            for filename, filename_raw in use.items():
                if self._raw_valid(os.path.join(self.location, filename_raw)):
                    d[filename] = self._get_raw(filename_raw)
        return d

    def set(self, key, value, force = True):
        # escape out slashes and other unsavory characters in a non
        # destructive way that won't work as a filename
        key_orig = key
        key = urllib.parse.quote(
            key, safe = '-_ ' + string.ascii_letters + string.digits)
        location = os.path.join(self.location, key)
        if value != None:
            # the storage is always a string, so encode what is not as
            # string as T:REPR, where T is type (b boolean, n number,
            # s string) and REPR is the textual repr, json valid
            if isinstance(value, bool):
                # do first, otherwise it will test as int
                value = "b:" + str(value)
            elif isinstance(value, numbers.Integral):
                # sadly, this looses precission in floats. A lot
                value = "i:%d" % value
            elif isinstance(value, numbers.Real):
                # sadly, this can loose precission in floats--FIXME:
                # better solution needed
                value = "f:%.10f" % value
            elif isinstance(value, str):
                if value.startswith("i:") \
                   or value.startswith("f:") \
                   or value.startswith("b:") \
                   or value.startswith("s:") \
                   or value == "":
                    value = "s:" + value
            else:
                raise ValueError("can't store value of type %s" % type(value))
            assert len(value) < 4096
        if value == None:
            # note that we are setting None (aka: removing the value)
            # we also need to remove any "subfield" -- KEY.a, KEY.b
            try:
                self._raw_unlink(location)
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            # FIXME: this can be optimized a lot, now it is redoing a
            # lot of work
            for key_itr in self.keys(key_orig + ".*"):
                key_itr_raw = urllib.parse.quote(
                    key_itr, safe = '-_ ' + string.ascii_letters + string.digits)
                location = os.path.join(self.location, key_itr_raw)
                try:
                    self._raw_unlink(location)
                except OSError as e:
                    if e.errno != errno.ENOENT:
                        raise
            return True	# already wiped by someone else
        if force == False:
            try:
                self._raw_write(location, value)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
                # ignore if it already exists
                return False
            return True

        # New location, add a unique thing to it so there is no
        # collision if more than one process is trying to modify
        # at the same time; they can override each other, that's
        # ok--the last one wins.
        location_new = location + "-" + str(os.getpid()) + "-" + str(threading.get_ident())
        rm_f(location_new)
        self._raw_write(location_new, value)
        self._raw_rename(location_new, location)
        return True

    def _get_raw(self, key, default = None):
        location = os.path.join(self.location, key)
        try:
            value = self._raw_read(location)
            # if the value was type encoded (see set()), decode it;
            # otherwise, it is a string
            if value.startswith("i:"):
                return json.loads(value.split(":", 1)[1])
            if value.startswith("f:"):
                return json.loads(value.split(":", 1)[1])
            if value.startswith("b:"):
                val = value.split(":", 1)[1]
                if val == "True":
                    return True
                elif val == "False":
                    return False
                raise ValueError("fsdb %s: key %s bad boolean '%s'"
                                 % (self.location, key, value))
            if value.startswith("s:"):
                # string that might start with s: or empty
                return value.split(":", 1)[1]
            return value	# other string
        except OSError as e:
            if e.errno == errno.ENOENT:
                return default
            raise

    def get(self, key, default = None):
        # escape out slashes and other unsavory characters in a non
        # destructive way that won't work as a filename
        key = urllib.parse.quote(
            key, safe = '-_ ' + string.ascii_letters + string.digits)
        return self._get_raw(key, default = default)

def field_needed(field, projections):
    """
    Check if the name *field* matches any of the *patterns* (ala
    :mod:`fnmatch`).

    :param str field: field name
    :param list(str) projections: list of :mod:`fnmatch` patterns
      against which to check field. Can be *None* and *[ ]* (empty).

    :returns bool: *True* if *field* matches a pattern in *patterns*
      or if *patterns* is empty or *None*. *False* otherwise.
    """
    if projections:
        # there is a list of must haves, check here first
        for projection in projections:
            if fnmatch.fnmatch(field, projection):
                return True	# we need this field
            # match projection a to fields a.[x.[y.[...]]]
            if field.startswith(projection + "."):
                return True
        return False		# we do not need this field
    else:
        return True	# no list, have it


from sqlalchemy import *
import json
from flask import Flask
from flask import request, jsonify
import os

keyT=os.path.join(os.getcwd(),'cc-ca.crt')
print(f"cockroachdb://moustafa:MoustafaHTN2021@free-tier.gcp-us-central1.cockroachlabs.cloud:26257/red-coyote-284.defaultdb?sslmode=verify-full&sslrootcert={keyT}")
engine = create_engine(f"cockroachdb://moustafa:MoustafaHTN2021@free-tier.gcp-us-central1.cockroachlabs.cloud:26257/red-coyote-284.defaultdb?sslmode=verify-full&sslrootcert={keyT}")
connection = engine.connect()
print(engine.table_names())
metadata = MetaData()

labels = Table('labels', metadata, autoload=True, autoload_with=engine)

def outputResults():
    result = []
    select_st = select([labels])
    tags_res_list = connection.execute(select_st)
    for res in tags_res_list:
        output = {
        "label":res.label,
        "start_time":float(res.start_time),
        "end_time":float(res.end_time),
        "confidence":float(res.confidence)
        }
        result.append(output)
    output_string = json.dumps(result)
    return output_string

def insertLabel(labelList):
    for l in labelList:
        info = json.loads(l);
        # insert data via insert() construct
        ins = labels.insert().values(
            label=info['label'],
            start_time = info['start_time'],
            end_time = info['end_time'],
            confidence = info['confidence'])
        res = connection.execute(ins)

def searchText(search1):
    result = []
    query_sql = text("SELECT * FROM labels WHERE label LIKE '%" + search1 + "%'")
    tags_res_list = (connection.execute(query_sql)).fetchall()
    for res in tags_res_list:
        output = {
        "label":res.label,
        "start_time":float(res.start_time),
        "end_time":float(res.end_time),
        "confidence":float(res.confidence)
        }
        result.append(output)
    output_string = json.dumps(result)
    return output_string

def searchTime(startTime, endTime):
    result = []
    query_sql = text("SELECT * FROM labels WHERE start_time >= " + str(startTime) + " AND end_time <= "+ str(endTime))
    tags_res_list = (connection.execute(query_sql)).fetchall()
    for res in tags_res_list:
        output = {
        "label":res.label,
        "start_time":float(res.start_time),
        "end_time":float(res.end_time),
        "confidence":float(res.confidence)
        }
        result.append(output)
    output_string = json.dumps(result)
    return output_string

app = Flask(__name__)
@app.route("/output")
def out():
    return outputResults()
@app.route("/search-text")
def searchTex():
    return searchText(str(request.args.get('text')))
@app.route("/search-time")
def searchTim():
    return searchTime(float(request.args.get('start')), float(request.args.get('end')))
@app.route("/")
def hello():
  return "Hello World!"

if __name__ == "__main__":
  app.run()


from keras.datasets import mnist
from keras.utils import np_utils
from tensorflow.python.keras.models import load_model

(X_train, y_train), (X_test, y_test) = mnist.load_data()
reconstructed_model = load_model("mnistSmall.h5")
X_test = X_test.astype('float32')
# Normalizing data
X_test /= 255
# one-hot encoding using keras numpy-related utilities
n_classes = 10
Y_test = np_utils.to_categorical(y_test, n_classes)
results = reconstructed_model.evaluate(X_test, Y_test, batch_size=128)
print("test loss, test acc:", results)


from bs4 import BeautifulSoup
from html.parser import HTMLParser
from html.entities import name2codepoint
import xlsxwriter

def parcing_html_table(table):
    table_dict = []
    row_index = 0
    for row in table.children:
        if row.name != 'tr':
            continue
        cell_index = 0
        if type(row).__name__ != 'NavigableString':
            table_dict.append([])
            table_dict[row_index] = {}
            table_dict[row_index]['context'] = 'table, tr'
            table_dict[row_index]['cells'] = []
            table_dict[row_index]['attrs'] = row.attrs

            for cell in row.children:
                if type(cell).__name__ != 'NavigableString':
                    table_dict[row_index]['cells'].append([])
                    table_dict[row_index]['cells'][cell_index] = {}
                    table_dict[row_index]['cells'][cell_index]['attrs'] = cell.attrs
                    values_index = 0
                    table_dict[row_index]['cells'][cell_index]['values'] = []
                    for cell_values in cell.contents:
                        cell_text = cell_values.string.replace('\n', '').strip()
                        if len(cell_text) > 0:
                            table_dict[row_index]['cells'][cell_index]['values'].append([])
                            table_dict[row_index]['cells'][cell_index]['values'][values_index] = {}
                            table_dict[row_index]['cells'][cell_index]['values'][values_index]['text'] = cell_text
                            if type(cell_values).__name__ != 'NavigableString':
                                cell_class = cell_values.attrs.get('class', None)
                                if cell_class:
                                    cell_class = cell_class[0]
                                table_dict[row_index]['cells'][cell_index]['values'][values_index]['context'] = cell_class
                            values_index += 1
                    cell_index += 1
            row_index += 1
    return table_dict

def parcing_html_styles(html):
    styles_dict = {}
    for tag in html:
        if tag.name == 'style':
            styles_name = ''
            styles_atrib_name = ''
            styles_atrib_values = ''
            what_we_read = 'styles_name'
            for tag_style_string in tag.children:
                for symbol in tag_style_string:
                    if symbol == '\n':
                        continue
                    elif what_we_read in ('styles_name', 'styles_atrib_name') and symbol == ' ':
                        continue
                    elif symbol == '{':
                        styles_name = styles_name.strip()
                        styles_dict[styles_name] = {}
                        what_we_read = 'styles_atrib_name'
                    elif symbol == ':':
                        styles_atrib_name = styles_atrib_name.strip()
                        styles_dict[styles_name][styles_atrib_name] = []
                        what_we_read = 'styles_atrib_values'
                    elif symbol == ';':
                        styles_atrib_values = styles_atrib_values.strip()
                        if ',' in styles_atrib_values:
                            styles_atrib_values = styles_atrib_values.split(',')
                        else:
                            styles_atrib_values = styles_atrib_values.split(' ')
                        styles_dict[styles_name][styles_atrib_name].append(styles_atrib_values)
                        styles_atrib_name = ''
                        styles_atrib_values = ''
                        what_we_read = 'styles_atrib_name'
                    elif symbol == '}':
                        styles_name = ''
                        what_we_read = 'styles_name'
                    else:
                        if what_we_read == 'styles_name':
                            styles_name += symbol
                        if what_we_read == 'styles_atrib_name':
                            styles_atrib_name += symbol
                        if what_we_read == 'styles_atrib_values':
                            styles_atrib_values += symbol
    return styles_dict

def table_to_excel(table_dict, cursor_start):
    def add_to_set(adding_cells_set, start_x, start_y, cell_end_x, cell_end_y):
        for i in range(start_x, cell_end_x + 1):
            for j in range(start_y, cell_end_y + 1):
                adding_cells_set.add((i, j))
        return adding_cells_set
    def get_cell_size(cell):
        rowspan = 1
        colspan = 1
        if cell['attrs'].get('rowspan', None):
            rowspan = int(cell['attrs'].get('rowspan', None))
        if cell['attrs'].get('colspan', None):
            colspan = int(cell['attrs'].get('colspan', None))
        return rowspan, colspan
    def calc_end_of_cell(cell_start_x, cell_start_y, rowspan, colspan):
        cell_end_x = cell_start_x + rowspan - 1
        cell_end_y = cell_start_y + colspan - 1
        return cell_end_x, cell_end_y
    def get_cell_text(cell, rowspan, colspan):
        cell_text = ''
        for i in cell['values']:
            if rowspan > 1:
                cell_text += i['text'] + str('\n')
            else:
                cell_text += i['text']
        return cell_text
    def write_cell(cell_start_x, cell_start_y, cell_end_x, cell_end_y, cell_text):
        if rowspan + colspan > 2:
            if rowspan > 1:
                cell_format = workbook.add_format()
                cell_format.set_text_wrap()
                cell_format.set_align('left')
                cell_format.set_align('top')
                cell_format.set_border()
                worksheet.merge_range(cell_start_x, cell_start_y, cell_end_x, cell_end_y, cell_text, cell_format)
            else:
                cell_format_two = workbook.add_format()
                cell_format_two.set_border()
                worksheet.merge_range(cell_start_x, cell_start_y, cell_end_x, cell_end_y, cell_text, cell_format_two)
        else:
            cell_format_two = workbook.add_format()
            cell_format_two.set_border()
            worksheet.write(cell_start_x, cell_start_y, cell_text, cell_format_two)
    def get_start_next_cell(cell_start_x, cell_start_y, cell_end_y, adding_cells_set):
        cell_start_y = cell_end_y
        while (cell_start_x, cell_start_y) in adding_cells_set:
            cell_start_y += 1
        return cell_start_y
    def get_start_next_cell_on_new_row(cell_start_x, cell_start_y):
        cell_start_y = 0
        cell_start_x += 1
        while (cell_start_x, cell_start_y) in adding_cells_set:
            cell_start_y += 1
        return cell_start_x, cell_start_y

    workbook = xlsxwriter.Workbook('hello.xlsx')
    worksheet = workbook.add_worksheet()
    cell_start_x = int(cursor_start[0])
    cell_start_y = int(cursor_start[1])
    adding_cells_set = set()

    for row in table_dict:
        for cell in row['cells']:
            rowspan, colspan = get_cell_size(cell)
            cell_end_x, cell_end_y = calc_end_of_cell(cell_start_x, cell_start_y, rowspan, colspan)
            adding_cells_set = set(add_to_set(adding_cells_set, cell_start_x, cell_start_y, cell_end_x, cell_end_y))
            cell_text = get_cell_text(cell, rowspan, colspan)
            write_cell(cell_start_x, cell_start_y, cell_end_x, cell_end_y, cell_text)
            cell_start_y = get_start_next_cell(cell_start_x, cell_start_y, cell_end_y, adding_cells_set)
        cell_start_x, cell_start_y = get_start_next_cell_on_new_row(cell_start_x, cell_start_y)
    workbook.close()

def main():
    with open('examples\only table.html', 'r', encoding='utf-8') as html_file_code:
        html_code = html_file_code.read()

    soup = BeautifulSoup(html_code, 'html.parser')

    table_dict = list(parcing_html_table(soup.table))
    cursor_start = (0, 0)
    table_to_excel(table_dict, cursor_start)

if __name__ == "__main__":
    main()


import os

import webapp2
from google.appengine.ext import db
import jinja2

import authentication

class Handler(webapp2.RequestHandler):
	template_dir = os.path.join(
			os.path.dirname(__file__), 'templates')
	jinja_env = jinja2.Environment(
		loader = jinja2.FileSystemLoader(template_dir),
		autoescape = True)

	def write(self, *a, **kw):
		'''Writes directly to the http request'''
		self.response.out.write(*a, **kw)

	def render_str(self, template, **params):
		'''Returns a string of the rendered template'''
		t = self.jinja_env.get_template(template)
		return t.render(params)

	def render(self, template, **kw):
		'''Render using the template and parameters'''
		username = authentication.getUserName(self)
		self.write(self.render_str(template,
			_user=username, _url=self.request.url, **kw))

	def throw_error(self, code):
		'''Throws an error, e.g. 404'''
		self.response.set_status(code)
		message = 'An error was encountered. Terribly sorry.'
		if code == 403:
			message = 'Access denied. You do not have permission to access this resource.'\
			' Please do not try again'
		if code == 404:
			message = 'The page or resource you requested could not be found. '\
			'If you caught a broken link, please report it to the naughty admin.'
		self.render('error.html', errorcode=code, errormessage=message)

class ErrorHandler(Handler):
	''''''
	def get(self, params):
		self.throw_error(404)

class SlashRedirect(Handler):
	'''Removes multiple trailing slashes and redirects'''
	def get(self, path):
		self.redirect(path)


# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    int_or_none,
    parse_duration,
    parse_iso8601,
    parse_resolution,
    url_or_none,
)

class CCMAIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ccma\.cat/(?:[^/]+/)*?(?P<type>video|audio)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.ccma.cat/tv3/alacarta/lespot-de-la-marato-de-tv3/lespot-de-la-marato-de-tv3/video/5630208/',
        'md5': '7296ca43977c8ea4469e719c609b0871',
        'info_dict': {
            'id': '5630208',
            'ext': 'mp4',
            'title': 'L\'espot de La Marató de TV3',
            'description': 'md5:f12987f320e2f6e988e9908e4fe97765',
            'timestamp': 1470918540,
            'upload_date': '20160811',
        }
    }, {
        'url': 'http://www.ccma.cat/catradio/alacarta/programa/el-consell-de-savis-analitza-el-derbi/audio/943685/',
        'md5': 'fa3e38f269329a278271276330261425',
        'info_dict': {
            'id': '943685',
            'ext': 'mp3',
            'title': 'El Consell de Savis analitza el derbi',
            'description': 'md5:e2a3648145f3241cb9c6b4b624033e53',
            'upload_date': '20171205',
            'timestamp': 1512507300,
        }
    }]

    def _real_extract(self, url):
        media_type, media_id = re.match(self._VALID_URL, url).groups()

        media = self._download_json(
            'http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={
                'media': media_type,
                'idint': media_id,
            })

        formats = []
        media_url = media['media']['url']
        if isinstance(media_url, list):
            for format_ in media_url:
                format_url = url_or_none(format_.get('file'))
                if not format_url:
                    continue
                label = format_.get('label')
                f = parse_resolution(label)
                f.update({
                    'url': format_url,
                    'format_id': label,
                })
                formats.append(f)
        else:
            formats.append({
                'url': media_url,
                'vcodec': 'none' if media_type == 'audio' else None,
            })
        self._sort_formats(formats)

        informacio = media['informacio']
        title = informacio['titol']
        durada = informacio.get('durada', {})
        duration = int_or_none(durada.get('milisegons'), 1000) or parse_duration(durada.get('text'))
        timestamp = parse_iso8601(informacio.get('data_emissio', {}).get('utc'))

        subtitles = {}
        subtitols = media.get('subtitols', {})
        if subtitols:
            sub_url = subtitols.get('url')
            if sub_url:
                subtitles.setdefault(
                    subtitols.get('iso') or subtitols.get('text') or 'ca', []).append({
                        'url': sub_url,
                    })

        thumbnails = []
        imatges = media.get('imatges', {})
        if imatges:
            thumbnail_url = imatges.get('url')
            if thumbnail_url:
                thumbnails = [{
                    'url': thumbnail_url,
                    'width': int_or_none(imatges.get('amplada')),
                    'height': int_or_none(imatges.get('alcada')),
                }]

        return {
            'id': media_id,
            'title': title,
            'description': clean_html(informacio.get('descripcio')),
            'duration': duration,
            'timestamp': timestamp,
            'thumbnails': thumbnails,
            'subtitles': subtitles,
            'formats': formats,
        }


import logging
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QWidget, QTextEdit, QVBoxLayout
from PyQt5.QtWidgets import QAction, QActionGroup, QMenu
from macro_manager import MacroMgr
from utils.logging_qt import QLogHandler
from utils.translator import string_to_level, level_to_string

class LogWidget(QWidget):
    def __init__(self, parent):
        super(LogWidget, self).__init__(parent)
        self._init_ui()
        # 绑定root logger
        self._log_handler = QLogHandler()
        self._log_handler.new_message.connect(self._text_edit.append)
        logging.getLogger().addHandler(self._log_handler)

    def _init_ui(self):
        self._text_edit = QTextEdit(self)
        self._text_edit.setReadOnly(True)
        lay = QVBoxLayout()
        lay.addWidget(self._text_edit)
        self.setLayout(lay)
        self._init_context_menu()

    def _init_context_menu(self):
        clear_act = QAction("清空日志", self)
        clear_act.triggered.connect(self._text_edit.clear)
        level_group = QActionGroup(self)
        level_menu = QMenu(self)
        level_menu.addAction(level_group.addAction(level_to_string(logging.DEBUG)))
        level_menu.addAction(level_group.addAction(level_to_string(logging.INFO)))
        level_menu.addAction(level_group.addAction(level_to_string(logging.WARNING)))
        level_menu.addAction(level_group.addAction(level_to_string(logging.ERROR)))
        for a in level_group.actions():
            a.setCheckable(True)
        level_group.triggered.connect(self._set_level)
        for action in level_group.actions():
            if string_to_level(action.text()) == MacroMgr.config().log_level:
                self._set_level(action)
                break
        log_level_act = QAction("日志等级", self)
        log_level_act.setMenu(level_menu)
        # 自定义右键菜单
        self._text_edit.setContextMenuPolicy(Qt.ContextMenuPolicy.ActionsContextMenu)
        self._text_edit.addAction(clear_act)
        self._text_edit.addAction(log_level_act)

    @staticmethod
    def _set_level(action: QAction):
        action.setChecked(True)
        MacroMgr.config().log_level = string_to_level(action.text())


import cv2 as cv

img = cv.imread('./Python/LearningOpenCv/lena.jpg', cv.IMREAD_GRAYSCALE)

print(img.shape)
print(img)
cv.imshow('Lena', img)
cv.waitKey(0)
cv.destroyAllWindows()

cv.imwrite('./Python/opencv/lena_gray.jpg', img)


from datetime import timedelta
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.utils.dates import days_ago

# DAG arguments
default_args = {
    'owner': 'Joe',
    'start_date': days_ago(0),
    'email': ['Joe@somemail.com'],
    'email_on_failure': True,
    'email_on_retry': True,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
}

# define the DAG
dag = DAG(
    dag_id='sample-etl-dag_1',
    default_args=default_args,
    description='Sample ETL DAG using Bash',
    schedule_interval=timedelta(days=1),
)

# define the tasks

# define the first task named extract
extract = BashOperator(
    task_id='extract',
    bash_command='echo "extract"',
    dag=dag,
)

# define the second task named transform
transform = BashOperator(
    task_id='transform',
    bash_command='echo "transform"',
    dag=dag,
)

# define the third task named load

load = BashOperator(
    task_id='load',
    bash_command='echo "load"',
    dag=dag,
)

# task pipeline
extract >> transform >> load


from django.db import models

# Create your models here.

class Persona(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)

    def full_name(self):
        return self.first_name + " " + self.last_name

    class Meta:
        abstract = True

class Profesor(Persona):

    salario = models.FloatField(default=0)
    class Meta:
        constraints = [
            models.UniqueConstraint(
                fields = ['id', 'first_name', 'last_name'], name = 'primary_key_profesor'
            )
        ]

class Salon(models.Model):
    aula = models.CharField(max_length=2)
    hora_entrada = models.TimeField()
    profesor_id = models.ForeignKey(Profesor, on_delete = models.CASCADE, default = 0)

class Alumno(Persona):
    idSalon = models.ForeignKey(Salon, on_delete = models.CASCADE)
    class Meta:
        constraints = [
            models.UniqueConstraint(
                fields = ['id', 'first_name', 'last_name'], name = 'primary_key_alumno'
            )
        ]

class OrderedAlum(Alumno):
    class Meta:
        ordering = ["last_name"]
        proxy = True

class Evaluacion(models.Model):
    hora_fecha = models.DateTimeField()
    curso = models.CharField(max_length=30)
    evaluador = models.CharField(max_length=50)

    class Meta:
        abstract = True

class Examen_Final(Evaluacion):
    duracion = models.IntegerField()
    n_preguntas = models.IntegerField()
    puntaje_total = models.IntegerField()

    def puntaje_pregunta(self):
        return self.puntaje_total / n_preguntas

class Proyecto(Evaluacion):
    tema_proyecto = models.CharField(max_length=100)
    n_grupos = models.IntegerField()

class OrderedNameProject(Proyecto):
    class Meta:
        proxy = True
        ordering = ["tema_proyecto"]


"""
Pelican BibTeX
==============

A Pelican plugin that populates the context with a list of formatted
citations, loaded from a BibTeX file at a configurable path.

The use case for now is to generate a ``Publications'' page for academic
websites.
"""
# Author: Vlad Niculae <vlad@vene.ro>
# Unlicense (see UNLICENSE for details)

from pelican import signals

import logging
logger = logging.getLogger(__name__)

import os
from operator import itemgetter
from docutils import nodes
from docutils.parsers.rst import directives, Directive

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    from pybtex.database.input.bibtex import Parser
    from pybtex.database.output.bibtex import Writer
    from pybtex.database import BibliographyData, PybtexError
    from pybtex.backends import html
    from pybtex.style.formatting import jo
except ImportError:
    logger.warn('`pelican_bibtex` failed to load dependency `pybtex`')

from jinja2 import Environment, FileSystemLoader
from pelican import signals

__version__ = '0.2.1'

class Publications(Directive):

    required_arguments = 1
    optional_arguments = 1
    has_content = False
    final_argument_whitespace = False
    option_spec = {
        # 'sort': sort,
        'template': directives.path,
        # 'strong': directives.unchanged,
    }

    def run(self):
        refs_file = self.arguments[0].strip()

        try:
            bibdata_all = Parser().parse_file(refs_file)
        except PybtexError as e:
            logger.warn('`pelican_bibtex` failed to parse file %s: %s' % (
                refs_file,
                str(e)))
            return

        # format entries
        jo_style = jo.Style()
        jo_style.strong = 'Razik'
        html_backend = html.Backend()
        formatted_entries = jo_style.format_entries(bibdata_all.entries.values())

        publications = []

        for formatted_entry in formatted_entries:
            key = formatted_entry.key
            entry = bibdata_all.entries[key]
            year = entry.fields.get('year')
            # This shouldn't really stay in the field dict
            # but new versions of pybtex don't support pop
            pdf = entry.fields.get('pdf', None)
            slides = entry.fields.get('slides', None)
            poster = entry.fields.get('poster', None)

            # render the bibtex string for the entry
            bib_buf = StringIO()
            bibdata_this = BibliographyData(entries={key: entry})
            Writer().write_stream(bibdata_this, bib_buf)
            text = formatted_entry.text.render(html_backend)

            publications.append((key,
                                 year,
                                 text,
                                 bib_buf.getvalue(),
                                 pdf,
                                slides,
                                 poster))

        # Load the publications template
        if 'template' in self.options:
            template_path = self.options['template']
            template_dir, template_name = os.path.split(template_path)
            env = Environment(loader=FileSystemLoader(template_dir))
            template = env.get_template(template_name)
        else:
            # Use template from the Pelican theme
            template = pelican_generator.get_template('publications')

        rendered_template = template.render(publications=publications)
        return [nodes.raw('', rendered_template, format='html')]

pelican_generator = None

def get_template_env(generator):
    global pelican_generator
    pelican_generator = generator

def register():
    # Register new RST directive
    directives.register_directive('publications', Publications)
    # Connect to Pelican generator init to get access
    # to the template environment.
    signals.generator_init.connect(get_template_env)


"""
In this code I create a recommendation system using hybrid method.The hybrid here uses model based
collaborative filtering as the base. To improve the performance of model based CF I performed data
preprocessing so as to have more information from the same data. I have performed Principal Component
Analysis to increase data interpretability and dimension reduction. I used only the 10 most important
features thus generated. Over model based CF, I then utitlised item based collaborative filtering. I
used prediction generared by item based as one of the feature to model based CF.

Error Distribution:
>=0 and <1: 105773
>=1 and <2: 34123
>=2 and <3: 6310
>=3 and <4: 790
>=4: 0

RSME: 0.9782285911303371
Duration: 861.13
"""

from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from pyspark import SparkContext, SparkConf
from datetime import datetime
from itertools import chain
import numpy as nmpy
import pandas as pnds
import json
import time
import math
import sys

NAN = [nmpy.nan]
result_test_data = dict()
result_train_data = dict()

def item_based(bid_uid_dblt, similarity):
    if bid_uid_dblt[1] not in training_data_uid:
        if bid_uid_dblt[0] in bid_actRates:
            temp = bid_actRates[bid_uid_dblt[0]]
            users_oth = list(temp.values())
            avrage_busId = sum(users_oth) / len(users_oth)
            return bid_uid_dblt, avrage_busId
        else:
            return bid_uid_dblt, 3.75

    rate_uid = training_data_uid[bid_uid_dblt[1]]

    ratings = []
    if bid_uid_dblt[0] not in bid_meanRates:
        for ind in rate_uid:
            ratings.append(rate_uid[ind])
        avrage_usrId = sum(ratings)/len(ratings)
        return bid_uid_dblt, avrage_usrId

    to_rate = bid_meanRates[bid_uid_dblt[0]]
    to_rate_set = set(to_rate.keys())

    sim_list = []
    rating_list = []

    for ind in rate_uid:
        ind_rate = (ind, rate_uid[ind])
        neighbor = bid_meanRates[ind_rate[0]]
        common_vals = set(neighbor.keys()) & to_rate_set
        if len(common_vals) < 3:
            continue
        dblt = tuple(sorted([ind_rate[0], bid_uid_dblt[0]]))
        if dblt in similarity:
            sim = similarity[dblt]
            sim_list.append(sim)
            rating_list.append(ind_rate[1])
        else:
            parta = 0
            partb = 0
            for intr in common_vals:
                parta += to_rate[intr] * to_rate[intr]
                partb += neighbor[intr] * neighbor[intr]
            parta = math.sqrt(parta)
            partb = math.sqrt(partb)
            sim_den = parta * partb
            if sim_den == 0:
                similarity[dblt] = 0
                continue
            sim = 0
            for intr in common_vals:
                sim += to_rate[intr] * neighbor[intr]
            sim = sim / sim_den
            if not sim < 0:
                sim = sim * (1 + 0.1)
            else:
                eph = (1 - 0.1) * abs(sim)
                sim += eph

            similarity[dblt] = sim
            sim_list.append(sim)
            rating_list.append(ind_rate[1])

    if sim_list == []:
        users_oth = list(bid_actRates[bid_uid_dblt[0]].values())
        avrage_busId = sum(users_oth)/len(users_oth)
        return bid_uid_dblt, avrage_busId
    elif len(sim_list) >= 5:
        sim_rating = sorted(tuple(zip(sim_list, rating_list)), key=lambda x: x[0])[0:5]
        den = 0
        num = 0
        for sr in sim_rating:
            den += abs(sr[0])
            num += sr[0] * sr[1]
        if num <= 25:
            temp = bid_actRates[bid_uid_dblt[0]]
            users_oth = list(temp.values())
            avrage_busId = sum(users_oth) / len(users_oth)
            return bid_uid_dblt, avrage_busId
        else:
            pred_val = num / den
            return bid_uid_dblt, pred_val
    else:
        temp = bid_actRates[bid_uid_dblt[0]]
        users_oth = list(temp.values())
        avrage_busId = sum(users_oth) / len(users_oth)
        return bid_uid_dblt, avrage_busId

"""
 Finds mean rating per business
 Finds normalised rating (actual rate - mean) and appends to res
 Appends actual rating to result
"""
def normalise_ratings(bid_listuid):
    total = 0
    for uids_strs in bid_listuid[1]:
        total += uids_strs[1]
    rBar = total / len(bid_listuid[1])

    res=[]
    for uids_strs in bid_listuid[1]:
        res.append((uids_strs[0], uids_strs[1]-rBar))

    k=[]
    for uids_strs in bid_listuid[1]:
        k.append((uids_strs[0], uids_strs[1]))
    return bid_listuid[0],dict(k),dict(res)

def get_matrix_train(val, uids_dfs, bids_dfs):
    list2 = [result_train_data[(val[0], val[1][0])]]
    if val[1][0] in uids_dfs.index:
        if val[0] in bids_dfs.index:
            list1 = list(uids_dfs.loc[val[1][0]])
            return  list1 + list(bids_dfs.loc[val[0]]) +  list2
        else:
            return list1 + NAN * bids_dfs.shape[1] + list2
    elif val[1][0] not in uids_dfs.index:
        shp = uids_dfs.shape[1]
        if val[0] in bids_dfs.index:
            return NAN * shp + list(bids_dfs.loc[val[0]]) + list2
    else:
        return NAN * shp + NAN * bids_dfs.shape[1] + list2

def get_matrix_test(val, uids_dfs, bids_dfs):
    uds = uids_dfs.shape[1]
    rtd = [result_test_data[val]]
    if val[1] in uids_dfs.index:
        if val[0] in bids_dfs.index:
            return list(uids_dfs.loc[val[1]]) + list(bids_dfs.loc[val[0]]) + rtd
        else:
            return list(uids_dfs.loc[val[1]]) + NAN * bids_dfs.shape[1] + rtd
    elif val[1] not in uids_dfs.index:
        if val[0] in bids_dfs.index:
            return NAN * uds + list(bids_dfs.loc[val[0]]) + rtd
    else:
        return NAN * uds + NAN * bids_dfs.shape[1] + rtd

def read_user(input_line):
    uid = input_line["user_id"]
    avg_str = input_line["average_stars"]
    rev_count = input_line["review_count"]
    fans = input_line["fans"]
    frnds = input_line["friends"]
    yelp_since = input_line["yelping_since"]
    useful = input_line["useful"]
    funny = input_line["funny"]
    cool = input_line["cool"]
    comp_hot = input_line["compliment_hot"]
    comp_more = input_line["compliment_more"]
    comp_prof = input_line["compliment_profile"]
    comp_cute = input_line["compliment_cute"]
    comp_lst = input_line["compliment_list"]
    comp_note = input_line["compliment_note"]
    comp_pl = input_line["compliment_plain"]
    comp_cool = input_line["compliment_cool"]
    comp_fun = input_line["compliment_funny"]
    comp_write = input_line["compliment_writer"]
    comp_ph = input_line["compliment_photos"]
    return uid, avg_str, rev_count, fans, frnds, yelp_since, useful, funny, cool, comp_hot, comp_more, comp_prof, comp_cute, comp_lst, comp_note, comp_pl, comp_cool, comp_fun, comp_write, comp_ph

def map_user(inpt):
    uid = user_ids_srno[inpt[0]]
    avg_str = float(inpt[1])
    rev_count = int(inpt[2])
    fans = int(inpt[3]) * int(inpt[3])
    frnds = 0
    if inpt[4] != "None":
        val = len(inpt[4].split(","))
        frnds = val * val
    yelp_since = (datetime.now() - datetime.strptime(inpt[5], "%Y-%m-%d")).days
    useful = int(inpt[6])
    funny = int(inpt[7])
    cool = int(inpt[8])
    comp_hot = int(inpt[9])
    comp_more = int(inpt[10])
    comp_prof = int(inpt[11])
    comp_cute = int(inpt[12])
    comp_lst = int(inpt[13])
    comp_note = int(inpt[14])
    comp_pl = int(inpt[15])
    comp_cool = int(inpt[16])
    comp_fun = int(inpt[17])
    comp_write = int(inpt[18])
    comp_ph = int(inpt[19])

    return uid, avg_str, rev_count, fans, frnds, yelp_since, useful, funny, cool, comp_hot, comp_more, comp_prof, comp_cute, comp_lst, comp_note, comp_pl, comp_cool, comp_fun, comp_write, comp_ph

def read_business(input_line):
    bid = input_line["business_id"]
    star = input_line["stars"]
    rev_count = input_line["review_count"]
    state = input_line["state"]
    cat = input_line["categories"]
    city = input_line["city"]
    lat = input_line["latitude"]
    longi = input_line["longitude"]
    hrs = input_line["hours"]
    return bid, star, rev_count, state, cat, city, lat, longi, hrs

def map_business(inpt):
    bid = business_ids_srno[inpt[0]]
    star = float(inpt[1])
    rev_count = int(inpt[2])
    state = inpt[3]
    cat = set()
    if inpt[4]:
        cat = set(inpt[4].split(", "))
    city = inpt[5]
    lat = float(inpt[6])
    longi = float(inpt[7])
    hrs = set()
    if inpt[8]:
        hrs = set(inpt[8].keys())
    return bid, star, rev_count, state, cat, city, lat, longi, hrs

def itemBasedCF():
    tdc = testing_data.collect()
    similarity = {}
    for val in tdc:
        bid_uid_dblt, pred_val = item_based(val, similarity)
        result_test_data[bid_uid_dblt] = pred_val

    inp_dta = training_data.map(lambda val: (val[0], val[1][0])).collect()
    for dblt in inp_dta:
        pair_y = dblt[1]
        pair_x = dblt[0]
        stars_training_data_uid = training_data_uid[pair_y].pop(pair_x)
        if training_data_uid[pair_y] == dict():
            training_data_uid.pop(pair_y)

        stars_mean_bid_meanRates = bid_meanRates[pair_x].pop(pair_y)
        if bid_meanRates[pair_x] == dict():
            bid_meanRates.pop(pair_x)

        stars_bid_actRates = bid_actRates[pair_x].pop(pair_y)
        if bid_actRates[pair_x] == dict():
            bid_actRates.pop(pair_x)

        bid_uid_dblt, pred_val = item_based(dblt, similarity)
        result_train_data[bid_uid_dblt] = pred_val

        if pair_y not in training_data_uid:
            training_data_uid[pair_y] = dict()
            training_data_uid[pair_y][pair_x] = stars_training_data_uid
        else:
            training_data_uid[pair_y][pair_x] = stars_training_data_uid

        if pair_x not in bid_meanRates:
            bid_meanRates[pair_x] = dict()
            bid_meanRates[pair_x][pair_y] = stars_mean_bid_meanRates
        else:
            bid_meanRates[pair_x][pair_y] = stars_mean_bid_meanRates

        if pair_x not in bid_actRates:
            bid_actRates[pair_x] = dict()
            bid_actRates[pair_x][pair_y] = stars_bid_actRates
        else:
            bid_actRates[pair_x][pair_y] = stars_bid_actRates

    print("Finished Item-based pred_val")

if __name__ == """__main__""":
    folder_path = sys.argv[1]
    test_path = sys.argv[2]
    st = time.time()
    conf = SparkConf().setMaster("local").set("spark.executor.memory", "4g").set("spark.driver.memory", "4g")
    sc = SparkContext(conf=conf)
    sc.setLogLevel("ERROR")

    # reading and preprocessing training data
    train_file = sc.textFile(folder_path + "yelp_train.csv")
    train_data_raw = train_file.filter(lambda val: not val.endswith("stars"))
    training_data = train_data_raw.map(lambda line: (line.split(',')[1], line.split(',')[0], line.split(',')[2]))
    busIds_train = training_data.map(lambda val: val[0]).distinct()
    distinct_bids_train = set(busIds_train.collect())
    uids_train = training_data.map(lambda val: val[1]).distinct()
    distinct_uids_train = set(uids_train.collect())

    # reading and preprocessing testing data
    test_file = sc.textFile(test_path)
    test_data_raw = test_file.filter(lambda val: not val.endswith("stars"))
    testing_data = test_data_raw.map(lambda line: (line.split(',')[1], line.split(',')[0]))
    bids_test = testing_data.map(lambda val: val[0]).distinct()
    distinct_bids_test = set(bids_test.collect())
    uid_test = testing_data.map(lambda val: val[1]).distinct()
    distinct_uids_test = set(uid_test.collect())

    business_ids = distinct_bids_train | distinct_bids_test
    business_ids_copy = business_ids
    srno_business_ids = dict(enumerate(business_ids))
    enum_business_ids_copy = enumerate(business_ids_copy)
    business_ids_srno = dict(map(lambda val:val[::-1], enum_business_ids_copy))

    user_ids = distinct_uids_train | distinct_uids_test
    user_ids_copy = user_ids
    srno_user_ids = dict(enumerate(user_ids))
    enum_user_ids_copy = enumerate(user_ids_copy)
    user_ids_srno = dict(map(lambda val:val[::-1], enum_user_ids_copy))

    training_data = training_data.map(lambda val: (business_ids_srno[val[0]], (user_ids_srno[val[1]], int(val[2][0])))).cache()
    testing_data =  testing_data.map(lambda val: (business_ids_srno[val[0]], user_ids_srno[val[1]])).cache()

    bid_uid_str_grpd = training_data.groupByKey()
    bid_uid_str_lst = bid_uid_str_grpd.map(lambda val: (val[0], list(val[1])))
    bid_uid_str_normalisd = bid_uid_str_lst.map(normalise_ratings).collect()
    a=[]
    b=[]
    c=[]
    for val in bid_uid_str_normalisd:
        a.append(val[0])
        b.append(val[1])
        c.append(val[2])
    bid_actRates = dict(zip(a,b))
    bid_meanRates = dict(zip(a,c))

    training_data_uid = training_data.map(lambda val: (val[1][0], (val[0], val[1][1]))).groupByKey()
    training_data_uid = training_data_uid.map(lambda val: (val[0], list(val[1])))
    training_data_uid = training_data_uid.map(lambda val: (val[0], dict(zip([v[0] for v in val[1]], [v[1] for v in val[1]])))).collect()
    a = []
    b = []
    for td in training_data_uid:
        a.append(td[0])
        b.append(td[1])
    training_data_uid = dict(zip(a, b))

    print("Performing Item based pred_val.")
    itemBasedCF()

    print("Reading files for Model based Collaborative Filtering.")
    usrIds = []
    with open(folder_path + "user.json") as file_reader:
        while True:
            data = file_reader.readlines(5000000)
            if not data:
                break
            uid_file_data = sc.parallelize(data).map(lambda val: json.loads(val))
            uid_file_data = uid_file_data.map(read_user)
            uid_file_data = uid_file_data.filter(lambda val: val[0] in user_ids_srno).map(map_user)
            usrIds += uid_file_data.collect()

    bid_file_data = sc.textFile(folder_path + "business.json")
    bid_file_data = bid_file_data.map(lambda val: json.loads(val))
    bid_file_data = bid_file_data.map(read_business)

    bid_file_data = bid_file_data.filter(lambda val: val[0] in business_ids_srno)

    bid_file_data = bid_file_data.map(map_business)
    busIds = bid_file_data.collect()

    cols_uid = ["user_id", "average_stars", "review_count", "fans_sqr", "friends_sqr", "yelping_since", "useful", "funny", "cool", "compliment_hot", "compliment_more", "compliment_profile", "compliment_cute", "compliment_list", "compliment_note", "compliment_plain", "compliment_cool", "compliment_funny", "compliment_writer", "compliment_photos"]
    cols_bid = ["business_id", "stars", "review_count", "state", "categories", "city", "latitude", "longitude", "hours"]
    uids_dfs = pnds.DataFrame(usrIds, columns = cols_uid)
    bids_dfs = pnds.DataFrame(busIds, columns = cols_bid)

    #-------------------------------------------
    print("Performing PCA.")
    princCompAnalysis = PCA(n_components = 5, svd_solver = 'full')
    xarr = uids_dfs.iloc[:, 9:]
    x = nmpy.array(xarr)
    print(x.shape)
    compl = princCompAnalysis.fit_transform(x)
    print(compl.shape)
    compl = nmpy.transpose(compl)
    len_compl = len(compl)
    for lc in range(len_compl):
        indx = "compliment_" + str(lc)
        uids_dfs[indx] = compl[lc]
    uids_dfs = uids_dfs.drop(columns = ["compliment_hot", "compliment_more", "compliment_profile", "compliment_cute", "compliment_list", "compliment_note", "compliment_plain", "compliment_cool", "compliment_funny", "compliment_writer", "compliment_photos"])

    print(bids_dfs.city.unique())
    city = pnds.get_dummies(bids_dfs.city, prefix = "city")
    city = nmpy.array(city)
    print(city.shape)
    princCompAnalysis = PCA(n_components = 10, svd_solver = 'full')
    city = princCompAnalysis.fit_transform(city)
    print(city.shape)
    city = nmpy.transpose(city)
    ctlen = len(city)
    for ct in range(ctlen):
        indx = "city_" + str(ct)
        bids_dfs[indx] = city[ct]
    bids_dfs = bids_dfs.drop(columns = ["city"])

    state = pnds.get_dummies(bids_dfs.state, prefix = "state")
    cols = [state.columns[-1]]
    state = state.drop(columns = cols)
    concat_lst = [bids_dfs, state]
    bids_dfs = pnds.concat(concat_lst, axis = 1)
    bids_dfs = bids_dfs.drop(columns = ["state"])

    set_cat = set()
    mat_cats = []
    for bidcat in bids_dfs.categories:
        set_cat = set(chain(set_cat, bidcat))
    for categs in set_cat:
        val1 = bids_dfs.categories.apply(lambda val: (val & set([categs])))
        val1 = val1.apply(lambda val: 0 if val == set() else 1)
        mat_cats.append(val1)
    mat_cats = nmpy.transpose(nmpy.array(mat_cats))
    princCompAnalysis = PCA(n_components=10, svd_solver='full')
    mat_cats = princCompAnalysis.fit_transform(mat_cats)
    mat_cats = nmpy.transpose(mat_cats)
    len_mc = len(mat_cats)
    for mc in range(len_mc):
        bids_dfs["category_" + str(mc)] = mat_cats[mc]
    bids_dfs = bids_dfs.drop(columns = ["categories"])

    weekdays = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
    days_mat = []
    for wd in weekdays:
        temp = bids_dfs.hours.apply(lambda val: (val & set([wd])))
        temp = temp.apply(lambda val: 0 if val == set() else 1)
        days_mat.append(temp)
    len_dm = len(days_mat)
    for ds in range(len_dm):
        ind = "Day_"+str(ds)
        bids_dfs[ind] = days_mat[ds]
    bids_dfs = bids_dfs.drop(columns=["hours"])

    print("Finished reading files for model-based CF")

    uids_dfs_pred = uids_dfs.set_index("user_id")
    bids_dfs_pred = bids_dfs.set_index("business_id")
    X = []
    for train_data in training_data.collect():
        temp = get_matrix_train(train_data, uids_dfs_pred, bids_dfs_pred)
        X.append(temp)

    X = nmpy.array(X)
    test_x_val = []
    for test_data in testing_data.collect():
        temp = get_matrix_test(test_data, uids_dfs_pred, bids_dfs_pred)
        test_x_val.append(temp)
    test_x_val = nmpy.array(test_x_val)
    temp = training_data.map(lambda val: val[1][1]).collect()
    Y = nmpy.array(temp)

    print(X.shape)
    print("Finished preparing X and Y")

    et = time.time()
    print(f"Duration: {et-st}")

    import xgboost as xgb
    from joblib import dump, load

    xgbr = load("model.md")

    estim_y_val = xgbr.predict(test_x_val)

    model_fin = ""
    rows = testing_data.collect()
    row_len = len(rows)
    for v in range(row_len):
        model_fin += srno_user_ids[rows[v][1]] + "," + srno_business_ids[rows[v][0]] + ","
        if estim_y_val[v] < 1:
            model_fin += str(1)
        elif estim_y_val[v] > 5:
            model_fin += str(5)
        else:
            model_fin += str(estim_y_val[v])
        model_fin += "\n"

    output = "user_id, business_id, stars\n"
    output += model_fin

    with open(sys.argv[3], "w") as out_file:
        out_file.writelines(output)

    et = time.time()
    print(f"Duration: {et-st}")


#!/usr/bin/env python3
import subprocess
import os
import sys
import json
sys.path.append("../lib/")
sys.path.append("../array/")

import json_parser
import pos
import cli
import test_result
import pos_constant
import DELETE_ARRAY_WITH_VOL

ARRAYNAME = DELETE_ARRAY_WITH_VOL.ARRAYNAME
DATA = "unvme-ns-0,unvme-ns-1,unvme-ns-2"
SPARE = "unvme-ns-3"

def check_result(detail):
    state = json_parser.get_state(detail)
    capacity = json_parser.get_capacity(detail)
    used =json_parser.get_used(detail)

    if state == "NORMAL" and capacity != 0 and used == 0 :
        return "pass"
    return "fail"

def set_result(detail):
    result = check_result(detail)
    code = json_parser.get_response_code(detail)
    with open(__file__ + ".result", "w") as result_file:
        result_file.write(result + " (" + str(code) + ")" + "\n" + out)

def execute():
    DELETE_ARRAY_WITH_VOL.execute()
    cli.create_array("uram0", DATA, SPARE, ARRAYNAME, "")
    cli.mount_array(ARRAYNAME)
    out = cli.array_info(ARRAYNAME)
    return out

if __name__ == "__main__":
    test_result.clear_result(__file__)
    out = execute()
    set_result(out)
    pos.kill_pos()


import requests
from flask import jsonify

from app.repositories.enrollment_repository import EnrollmentRepository

class EnrollmentService:
    def enroll_student_in_class(student_id, subject_num, class_num):
        student_response = requests.get(f'http://student-app:80/{student_id}')
        if student_response.status_code != 200:
            return jsonify({'error': 'Failed to retrieve student information.'}), 500

        subject_response = requests.get(
            f'http://subject-app:80/{subject_num}&{class_num}')
        if subject_response.status_code != 200:
            return jsonify({'error': 'Failed to retrieve subject information.'}), 500

        enrollment = EnrollmentRepository.create_enrollment(
            subject_num=subject_num,
            class_num=class_num,
            student_id=student_id
        )

        return enrollment


"""Mapping Interface"""
import glob
import os
from io import BytesIO

import numpy as np
import seaborn as sns
from paste.request import parse_formvars
from pydep.io.wepp import read_slp
from pyiem.plot.use_agg import plt

def make_plot(huc12, scenario):
    """Make the map"""
    os.chdir("/i/%s/slp/%s/%s" % (scenario, huc12[:8], huc12[8:]))
    res = []
    for fn in glob.glob("*.slp"):
        try:
            slp = read_slp(fn)
        except Exception:
            print(f"read_slp failed for {fn}")
            continue
        bulk = (slp[-1]["y"][-1]) / slp[-1]["x"][-1]
        length = slp[-1]["x"][-1]
        if bulk < -1:
            print("Greater than 100%% slope, %s %s" % (fn, bulk))
            continue
        res.append([(0 - bulk) * 100.0, length])

    data = np.array(res)
    g = sns.jointplot(
        x=data[:, 1], y=data[:, 0], s=40, zorder=1, color="tan"
    ).plot_joint(sns.kdeplot, n_levels=6)
    g.ax_joint.set_xlabel("Slope Length [m]")
    g.ax_joint.set_ylabel("Bulk Slope [%]")
    g.fig.subplots_adjust(top=0.8, bottom=0.2, left=0.15)
    g.ax_joint.grid()
    g.ax_marg_x.set_title(
        ("HUC12 %s DEP Hillslope\n" "Kernel Density Estimate (KDE) Overlain")
        % (huc12,),
        fontsize=10,
    )

    ram = BytesIO()
    plt.gcf().set_size_inches(3.6, 2.4)
    plt.savefig(ram, format="png", dpi=100)
    ram.seek(0)
    return ram.read()

def application(environ, start_response):
    """Do something fun"""
    form = parse_formvars(environ)
    huc12 = form.get("huc12", "000000000000")[:12]
    scenario = int(form.get("scenario", 0))

    start_response("200 OK", [("Content-type", "image/png")])
    return [make_plot(huc12, scenario)]


from django.db import models
from django.dispatch import receiver
from django.db.models.signals import post_save
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

class Info(models.Model):
    user = models.OneToOneField(User, verbose_name='auth_user')

    first_name = models.CharField(_('first name'), max_length=30, blank=True)
    last_name = models.CharField(_('last name'), max_length=30, blank=True)
    email = models.EmailField(_('email address'), blank=True)
    phone = models.CharField(_('phone'), max_length=20, blank=True)

    avatar_path = models.CharField(
        _('avatar path'), max_length=255, blank=True)
    background_path = models.CharField(
        _('background path'), max_length=255, blank=True)

    def serialized_user(self):
        u = UserSerializer(instance=self.user)
        return u.data

    def followers(self):
        objs = self.user.followerships.filter(
            is_active=True, is_confirm=True)
        # temporary test
        followers_id = []
        for obj in objs:
            followers_id.append(obj.follower_id)
        return followers_id

    def followeds(self):
        objs = self.user.followedships.filter(
            is_active=True, is_confirm=True)
        # temporary test
        followeds_id = []
        for obj in objs:
            followeds_id.append(obj.followed_id)
        return followeds_id

    def follow(self, followed_id):
        follower = self.user
        if follower.id == int(followed_id):
            return 'Cannot Follow Yourself'
        try:
            followed = User.objects.get(pk=followed_id)
        except User.DoesNotExist:
            return 'User DoesNotExist'
        obj = Friendship.objects.get_or_create(
            followed=followed, follower=follower)
        if obj[1]:
            if followed.setting.need_permission_to_listen:
                obj[0].is_confirm = False
                obj[0].save()
            else:
                obj[0].active_dateline = obj[0].dateline
                obj[0].save()
            return 'Success'
        else:
            if not obj[0].is_active:
                obj[0].is_active = True
                obj[0].save()
                return 'Success'
            else:
                return 'Already Exist'

    def confirm(self, follower_id):
        try:
            obj = self.user.followerships.get(follower_id=follower_id)
            obj.confirm()
            return 'Success'
        except Friendship.DoesNotExist:
            return 'Friendship DoesNotExist'

    def unfollow(self, followed_id):
        try:
            obj = self.user.followedships.get(
                followed_id=followed_id)
            obj.delete()
            return 'Success'
        except Friendship.DoesNotExist:
            return 'Friendship DoesNotExist'

    def remove_follower(self, follower_id):
        try:
            obj = self.user.followerships.get(
                follower_id=follower_id)
            obj.delete()
            return 'Success'
        except Friendship.DoesNotExist:
            return 'Friendship DoesNotExist'

@receiver(post_save, sender=User)
def user_create_info(sender, **kwargs):
    if kwargs.get('created'):
        i = kwargs.get('instance')
        info = Info(
            user=i,
            first_name=i.first_name,
            last_name=i.last_name,
            email=i.email,
            phone='',
            avatar_path='',
            background_path=''
        )
        info.save()
    else:
        i = kwargs.get('instance')
        info = Info.objects.get(user=i)
        info.first_name = i.first_name
        info.last_name = i.last_name
        info.email = i.email
        info.save()

class Setting(models.Model):
    user = models.OneToOneField(User, verbose_name=_('auth_user'))
    need_permission_to_listen = models.BooleanField(
        default=False, verbose_name=_('need permission to listen'))
    need_permission_to_message = models.BooleanField(
        default=False,  verbose_name=_('need permission to msg'))

@receiver(post_save, sender=User)
def user_create_setting(sender, **kwargs):
    if kwargs.get('created'):
        i = kwargs.get('instance')
        setting = Setting(
            user=i,
            need_permission_to_message=False,
            need_permission_to_listen=False
        )
        setting.save()

class Friendship(models.Model):
    followed = models.ForeignKey(User, related_name='followerships')
    follower = models.ForeignKey(User, related_name='followedships')
    dateline = models.DateTimeField(auto_now_add=True)
    is_confirm = models.BooleanField(default=True)
    is_active = models.BooleanField(default=True)
    active_dateline = models.DateTimeField(blank=True, null=True)

    class Meta:
        unique_together = ('followed', 'follower')

    def __unicode__(self):
        return "%s listen to %s" % (self.follower, self.followed)

    def confirm(self):
        '''only followed user has permission'''
        self.is_confirm = True
        self.save()

    def delete(self):
        '''followed and follower user both have permission'''
        self.is_active = False
        self.save()

from users.serializers import UserSerializer


#!/usr/bin/env python
# coding: utf-8

# ## Downloading the Dataset
#
# Download and extract the 'yelp_review_full_csv.tar.gz' file from https://drive.google.com/drive/u/0/folders/0Bz8a_Dbh9Qhbfll6bVpmNUtUcFdjYmF2SEpmZUZUcVNiMUw1TWN6RDV3a0JHT3kxLVhVR2M
#
# Make sure that 'train.csv' and 'test.csv' files are present in this directory

# In[11]:

import os

assert os.path.exists('train.csv') and os.path.exists('test.csv')

# In[13]:

get_ipython().system('export PYTHONIOENCODING=utf8')

# In[14]:

import nltk
import pandas as pd
import csv
import sys
import spacy
import re
import random
import codecs
from importlib import reload

random.seed(1357)
def read_input_file(input_file):
    lines = csv.reader(codecs.open(input_file, "r", encoding="utf-8"))
    lines = list(lines)
    random.shuffle(lines)
    new_labels = []
    new_lines = []
    for label, line in lines:
        if int(label) < 3:
            new_labels.append("0")
            new_lines.append(line)
        elif int(label) > 3:
            new_labels.append("1")
            new_lines.append(line)

    print (new_labels[:2], new_lines[:2])
    print(len(new_labels), len(new_lines))
    return new_labels, new_lines

# In[15]:

labels_train, content_train = read_input_file("train.csv")
assert(len(labels_train) == len(content_train))
print (len(labels_train))

labels_dev, content_dev = labels_train[:7000], content_train[:7000]
keys_dev = ["dev"]* len(labels_dev)

labels_train, content_train = labels_train[7000:], content_train[7000:]
keys_train = ["train"]*len(labels_train)

# In[16]:

labels_test, content_test = read_input_file("test.csv")
keys_test = ["test"]*len(labels_test)
assert(len(labels_test) == len(content_test))
print (len(labels_test))

# In[17]:

content_train[:10]

# In[ ]:

nlp = spacy.load('en', disable=['parser', 'tagger', 'ner'])

def tokenize(text) :
    #text = " ".join(text)
    text = text.replace("-LRB-", '')
    text = text.replace("-RRB-", " ")
    text = text.strip()
    tokens = " ".join([t.text.lower() for t in nlp(text)])
    return tokens

labels = [int(i) for i in labels_train]
content = [tokenize(i) for i in content_train]

assert(len(labels) == len(content))
labels[:3]
content[:3]

# In[ ]:

content[0]
labels = labels_train + labels_dev + labels_test
content = content_train + content_dev + content_test
keys = keys_train + keys_dev + keys_test

# In[ ]:

df = pd.DataFrame({'text' : content, 'label' : labels, 'exp_split' : keys})
df.to_csv('yelp_dataset.csv', index=False)

df = {'paragraph' : df_paragraphs, 'question' : df_questions, 'answer' : df_answers, 'exp_split' : df_exp_splits}
df = pd.DataFrame(df)

# In[2]:

get_ipython().run_line_magic('run', '"../preprocess_data_BC.py" --data_file yelp_dataset.csv --output_file ./vec_yelp.p --word_vectors_type fasttext.simple.300d --min_df 20')

# In[ ]:


from typing import List

class Solution:
    def canPlaceFlowers(self, flowerbed: List[int], n: int) -> bool:
        max_place = 0
        count = len(flowerbed)
        if count == 0:
            return n == 0
        if count == 1:
            return flowerbed[0] + n < 2

        if flowerbed[1] == 0 and flowerbed[0] == 0:
            max_place += 1
            flowerbed[0] = 1

        for i in range(1, count - 1):
            if flowerbed[i - 1] == 0 and flowerbed[i] == 0 and flowerbed[i + 1] == 0:
                flowerbed[i] = 1
                max_place += 1
                i += 1

        if flowerbed[-1] == 0 and flowerbed[-2] == 0:
            max_place += 1

        return n <= max_place

print(Solution().canPlaceFlowers(flowerbed=[0, 0,0,0, 1], n=2))


import os

import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from bs4 import BeautifulSoup
import re

def count_double_negatives(text):
    sentences = sent_tokenize(text)
    double_negatives_count = 0

    for sentence in sentences:
        if has_double_negatives(sentence):
            double_negatives_count += 1

    return double_negatives_count

def has_double_negatives(sentence):
    # Match double negatives pattern
    pattern = r"\b(?:not|n't|never|no)\s+\b(?:\w+\s+)?(?:not|n't|never|no)\b"
    return re.search(pattern, sentence, re.I) is not None

def count_obscure_language_qualifiers(text):
    sentences = sent_tokenize(text)
    obscure_qualifiers_count = 0
    obscure_words = ["possibly", "perhaps", "maybe", "might", "could", "potentially","may"]

    for sentence in sentences:
        if has_obscure_qualifiers(sentence, obscure_words):
            obscure_qualifiers_count += 1

    return obscure_qualifiers_count

def has_obscure_qualifiers(sentence, obscure_words):
    words = nltk.word_tokenize(sentence.lower())
    return any(word in obscure_words for word in words)

def list_files(directory):
    files = []
    for filename in os.listdir(directory):
        filepath = os.path.join(directory, filename)
        if os.path.isfile(filepath):
            files.append(filepath)
    return files

if __name__ == "__main__":
    folder_path = "./pp_example/"
    files_in_folder = list_files(folder_path)
    for file in files_in_folder:
        if file == "./pp_example/.DS_Store":
            continue
        with open(file, "r", encoding="utf-8") as file:
            html_content = file.read()

        soup = BeautifulSoup(html_content, "html.parser")
        text = soup.get_text()

        double_negatives_count = count_double_negatives(text)
        obscure_qualifiers_count = count_obscure_language_qualifiers(text)
        print("Double Negatives sentences Count:", double_negatives_count)
        print("Obscure Language Qualifiers sentences Count:", obscure_qualifiers_count)
        print("------------------")


import os
import logging
from logging.handlers import RotatingFileHandler
from logging.config import dictConfig
from datetime import datetime

from flask import Flask, url_for
try:
    from flask_restplus import Api
except ImportError:
    import werkzeug
    werkzeug.cached_property = werkzeug.utils.cached_property
    from flask_restplus import Api

def create_app():

    dictConfig({
    'version': 1,
    'formatters': {'default': {
        'format': '[%(asctime)s] %(levelname)s: %(message)s',
    }},
    'handlers': {'wsgi': {
        'class': 'logging.StreamHandler',
        'stream': 'ext://flask.logging.wsgi_errors_stream',
        'formatter': 'default'
    }},
    'root': {
        'level': 'INFO',
        'handlers': ['wsgi']
    }
    })

    app = Flask(__name__)
    app.config['SECRET_KEY'] = os.urandom(32)

    from views import main_views, api_views
    app.register_blueprint(main_views.bp)
    app.register_blueprint(api_views.bp)
    return app

if __name__ == '__main__':
    app = create_app()
    app.run(host='127.0.0.1', debug=True)


import typing
from typing import Any

from inmanta_module_factory import builder, inmanta

from terraform_module_generator.schema.helpers.cache import cache_method_result

from .nested_block import NestedBlock, nested_block

def is_block_map(schema: Any) -> bool:
    return schema.nesting == 4

@nested_block(index="abc-map-z", condition=is_block_map)
class MapNestedBlock(NestedBlock):
    @cache_method_result
    def get_map_key_attribute(
        self, module_builder: builder.InmantaModuleBuilder
    ) -> inmanta.Attribute:
        return inmanta.Attribute(
            name="_map_key",
            inmanta_type=inmanta.InmantaStringType,
            optional=False,
            description="The key used to reference this entity in the mapping it is a part of.",
            entity=self.get_entity(module_builder),
        )

    @cache_method_result
    def get_config_block_attributes(
        self,
        module_builder: builder.InmantaModuleBuilder,
        imports: typing.Set[str],
    ) -> typing.Dict[str, str]:
        attributes = super().get_config_block_attributes(module_builder, imports)
        attributes["nesting_mode"] = '"map"'
        attributes["key"] = "self." + self.get_map_key_attribute(module_builder).name
        return attributes

    @cache_method_result
    def get_entity_index(
        self, module_builder: builder.InmantaModuleBuilder
    ) -> typing.Optional[inmanta.Index]:
        index = inmanta.Index(
            path=self.get_entity(module_builder).path,
            entity=self.get_entity(module_builder),
            fields=[
                self.get_entity_relation(module_builder).peer,
                self.get_map_key_attribute(module_builder),
            ],
            description="This index ensure that each element of the config tree is unique",
        )
        module_builder.add_module_element(index)
        return index


from PyQt5.QtWidgets import QWidget, QLabel, QPushButton, QHBoxLayout
from PyQt5.QtGui import QFont
from PyQt5.QtCore import Qt, QPoint

class MyCustomTitleBar (QWidget):

    def __init__(self, parent=None):

        super (MyCustomTitleBar, self).__init__(parent)
        self.myParent = parent

        self.setStyleSheet(
            "QPushButton {max-width:20px; min-height:20px"
            "color:#4fc3f7;"
            "background-color:#424242;"
            #"border:2px solid #4fc3f7;"

            "border-radius:16px;"
            "font-size:35px;"
            "font-weight:bold;}"
            +
            "QPushButton:hover {color:#212121;"
            "background-color:#4fc3f7;}"
            +
            "QPushButton:pressed {color:white;"
            "background-color:#212121;"
            "border-color:white;}")

        #타이틀
        self.title = QLabel("HMemo")
        self.title.setFixedHeight (20)
        self.title.setAlignment (Qt.AlignCenter)

        #self.title.setStyleSheet (f"max-width: g_screen_with)px; background-color:#212121; color: rgb(187, 187, 187);")
        self.title.setStyleSheet ("background-color: #424242; color: rgb(187, 187, 187);")

        self.myFont = QFont("맑은 고딕", 12)
        self.title.setFont(self.myFont)

        self.title.mouseDoubleClickEvent = self.title_mouseDoubleclickEvent # 추가

        btn_size = 35

        # 최소화 버튼
        self.btn_min = QPushButton("_")
        self.btn_min.clicked.connect(self.btn_min_clicked)
        self.btn_min.setFixedSize(btn_size, btn_size)
        self.btn_min.setStyleSheet("background-color:# 424242;")

        # 최대화 버튼
        self.btn_max = QPushButton("□")
        self.btn_max.clicked.connect(self.btn_max_clicked)
        self.btn_max.setFixedSize(btn_size, btn_size)

        self.btn_max.setStyleSheet("background-color:# 424242;")

        # 닫기 버튼
        self.btn_close = QPushButton("X")
        self.btn_close.clicked.connect(self.btn_close_clicked)
        self.btn_close.setFixedSize(btn_size, btn_size)

        self.btn_close.setStyleSheet("background-color:# 424242;")
        lay = QHBoxLayout(self)
        lay.setContentsMargins(0, 0, 0, 0)
        lay.addWidget(self.title)
        lay.addWidget(self.btn_min)
        lay.addWidget(self.btn_max)
        lay.addWidget(self.btn_close)

        self.pressing = False
        self.dragPosition = QPoint()

    def mousePressEvent(self, event):
        self.start = event.globalPos()
        self.pressing = True

    def mouseMoveEvent(self, event):

        if self.pressing:
            self.end = event.globalPos()
            delta = self.end - self.start
            self.window().move(self.window().pos() + delta)
            self.start = self.end

    def mouseReaseEvent(self, QMouseEvent):
        self.pressing = False

    def resizeEvent(self, ResizeEvent):
        # super (TitleBar, self).resize Event (QResizeEvent)
        # self.title. setFixedwidth (self.window).width())
        pass

    def btn_close_clicked(self):
        # self.window().close()
        self.myParent.close()

    def btn_min_clicked(self):
        self.window().showMinimized()

    def btn_max_clicked(self):

        btnText = self.btn_max.text()

        if btnText == "□":
            self.btn_max.setText("▣")
            self.window().showMaximized()

        else:
            self.btn_max.setText("□")
            self.window().showNormal()

    def title_mouseDoubleclickEvent(self, QMouseEvent):
        self.btn_max_clicked()


# -*- coding: utf-8 -*-
#########################################################
# python
import os
import sys
import traceback
import time
from datetime import datetime, timedelta
import logging
import urllib
import re
import json

# third-party
import requests
from flask import Blueprint, request, Response, send_file, render_template, redirect, jsonify
from sqlalchemy import desc
from lxml import html

# sjva 공용
from framework import db, scheduler
from framework.job import Job
from framework.util import Util
from system import SystemLogicTrans
import framework.common.fileprocess as FileProcess

# 패키지
from .plugin import logger, package_name
from .model import ModelSetting
#########################################################

class LogicNormal(object):

    @staticmethod
    def proxy_init():
        try:
            logger.debug(FileProcess.Vars.proxies)
            if ModelSetting.get_bool('use_proxy'):
                tmp = ModelSetting.get('proxy_url')
                FileProcess.Vars.proxies = {
                    "http"  : tmp,
                    "https" : tmp,
                }
            else:
                FileProcess.Vars.proxies = None
            logger.debug(FileProcess.Vars.proxies)
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())

    ########################################################

    @staticmethod
    def test(req):
        try:
            target = req.form['target']
            keyword = req.form['test']
            if target == 'dmm':
                return LogicNormal.test_dmm(keyword)
            elif target == 'javdb':
                return LogicNormal.test_javdb(keyword)
            elif target == 'avgle':
                return LogicNormal.test_avgle(keyword)
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())

    @staticmethod
    def test_dmm(keyword):
        try:
            ret = {}
            ret['search'] = FileProcess.dmm_search(keyword)
            if len(ret['search']) == 1:
                ret['update'] = FileProcess.dmm_update(ret['search'][0]['id'], use_discord_proxy=ModelSetting.get_bool('use_discord_proxy'))
            else:
                for tmp in ret['search']:
                    if tmp['score'] == 100:
                        ret['update'] = FileProcess.dmm_update(tmp['id'])
                        break
            return ret
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())

    @staticmethod
    def test_javdb(keyword):
        try:
            ret = {}
            ret['search'] = FileProcess.javdb_search(keyword)
            if len(ret['search']) == 1:
                ret['update'] = FileProcess.javdb_update(ret['search'][0]['id'])
            else:
                for tmp in ret['search']:
                    if tmp['score'] == 100:
                        ret['update'] = FileProcess.javdb_update(tmp['id'])
                        break
            return ret
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())
            return False

    @staticmethod
    def test_avgle(keyword):
        try:
            ret = {}
            ret['search'] = LogicNormal.avgle_search(keyword)

            if len(ret['search']['response']['videos']) >= 1:
                ret['update'] = LogicNormal.avgle_update(ret['search']['response']['videos'][0]['vid'])
            return ret
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())
            return False

    @staticmethod
    def avgle_search(keyword):
        try:
            session = requests.Session()
            url = 'https://api.avgle.com/v1/jav/%s/0?limit=10' % keyword
            response = session.get(url, headers=LogicNormal.headers, proxies=LogicNormal.proxies)
            data = response.json()
            return data
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())
            return False

    @staticmethod
    def avgle_update(vid):
        try:
            session = requests.Session()
            url = 'https://api.avgle.com/v1/video/%s' % vid
            response = session.get(url, headers=LogicNormal.headers, proxies=LogicNormal.proxies)
            data = response.json()
            return data
        except Exception as e:
            logger.error('Exception:%s', e)
            logger.error(traceback.format_exc())
            return False

    ########################################################