from typing import Callable, Optional, Sequence, Tuple, TypeVar, Union

import tensorflow as tf

from multi_graph.tf_typing import TensorLike, TensorLikeSpec

T = TypeVar("T")
NameLike = Union[str, tf.Tensor]

def _spec_to_placeholder(spec, name=None):
    if isinstance(spec, tf.TensorSpec):
        return tf.keras.backend.placeholder(
            shape=spec.shape, dtype=spec.dtype, name=name
        )
    if isinstance(spec, tf.SparseTensorSpec):
        return tf.keras.backend.placeholder(
            shape=spec.shape, dtype=spec.dtype, sparse=True, name=name
        )
    if isinstance(spec, tf.RaggedTensorSpec):
        return tf.keras.backend.placeholder(
            shape=spec._shape,  # pylint:disable=protected-access
            dtype=spec._dtype,  # pylint:disable=protected-access
            ragged=True,
            name=name,
        )
    raise TypeError(f"Invalid type for spec: must be TensorSpecLike, got {spec}")

def _spec_to_input(spec, name=None):
    if isinstance(spec, tf.RaggedTensorSpec):
        shape = spec._shape  # pylint:disable=protected-access
        dtype = spec._dtype  # pylint:disable=protected-access
    else:
        shape = spec.shape
        dtype = spec.dtype
    inp = tf.keras.Input(
        shape=shape[1:],
        batch_size=shape[0],
        ragged=isinstance(spec, tf.RaggedTensorSpec),
        sparse=isinstance(spec, tf.SparseTensorSpec),
        dtype=dtype,
        name=name,
    )
    if isinstance(spec, tf.RaggedTensorSpec):
        assert isinstance(inp, tf.RaggedTensor)
    return inp

def _batched_placeholder_like(x, batch_size=None, name=None):
    shape = [batch_size, *x.shape]
    if isinstance(x, tf.RaggedTensor):
        for i in range(x.ragged_rank):
            shape[i + 2] = None

    out = tf.keras.backend.placeholder(
        shape=shape,
        dtype=x.dtype,
        sparse=isinstance(x, tf.SparseTensor),
        ragged=isinstance(x, tf.RaggedTensor),
        name=name,
    )
    if isinstance(x, tf.RaggedTensor):
        assert isinstance(out, tf.RaggedTensor)
    return out

def _placeholder_like(x, name=None):
    out = tf.keras.backend.placeholder(
        shape=x.shape,
        dtype=x.dtype,
        sparse=isinstance(x, tf.SparseTensor),
        ragged=isinstance(x, tf.RaggedTensor),
        name=name,
    )
    if isinstance(x, tf.RaggedTensor):
        assert isinstance(out, tf.RaggedTensor)
    return out

def flatten_inputs(fn, input_structure, expand_composites=False):
    """
    Change the input interface of the given function.

    Args:
        fn: function with signature `fn(*args)`.
        input_structure: different input signature
        expand_composites: used in `tf.nest.flatten`.

    Returns:
        function with signature `out_fn(inputs)`, where `inputs` must have the
            same structure as `input_structure` according to
            `tf.nest.assert_same_structure`.
    """

    def flat_fn(*inputs):
        tf.nest.assert_same_structure(
            inputs, input_structure, expand_composites=expand_composites
        )
        flat_args = tf.nest.flatten(inputs, expand_composites=expand_composites)
        return fn(*flat_args)

    return flat_fn

def repack_outputs(fn, output_structure, expand_composites=False):
    """
    Change the output interface of a given function.

    Args:
        fn: function with signature `fn(*args, **kwargs) -> Sequence`
        output_structure: desired output structure.
        expand_composites: whether outputs of `fn` have composites that should
            be reduced via `tf.nest.pack_sequence_as`.

    Returns:
        function with signature 'out_fn(*args, **kwargs) -> outupts`, where
            outputs has structure of `output_structure`.
    """

    def flat_fn(*args, **kwargs):
        out = fn(*args, **kwargs)
        return tf.nest.pack_sequence_as(
            output_structure, out, expand_composites=expand_composites
        )

    return flat_fn

def subgraph(graph_def, inputs, outputs) -> Callable:
    """
    Extract a subgraph from the given graph_def as a `@tf.function`ed callable.

    Args:
        graph_def: a `GraphDef`, like from `tf.Graph.as_graph_def`.
        inputs: tensors or op names as inputs.
        outputs: tensor or tensor names of outputs.

    Returns:
        A callable which maps f(*inputs) to a list of tensors given in outputs.
    """
    input_op_names = tuple(
        t if isinstance(t, str) else t.op.name
        for t in tf.nest.flatten(inputs, expand_composites=True)
    )
    output_names = tuple(
        t if isinstance(t, str) else t.name
        for t in tf.nest.flatten(outputs, expand_composites=True)
    )

    @tf.function()
    def graph_fn(*args, **kwargs):
        args = tf.nest.flatten((args, kwargs), expand_composites=True)
        if len(args) != len(input_op_names):
            raise ValueError(
                f"Expected {len(input_op_names)} args, got {len(args)}: {args}"
            )
        assert len(args) == len(input_op_names)
        input_map = dict(zip(input_op_names, args))
        flat_out = tf.graph_util.import_graph_def(
            graph_def, input_map=input_map, return_elements=output_names
        )
        return tf.nest.pack_sequence_as(outputs, flat_out, expand_composites=True)

    return graph_fn

class GraphBuilder:
    def __init__(self, name="graph_builder"):
        self._graph = tf.Graph()
        self._outputs = []
        self._inputs = []
        self._ctxs = []
        self._name = name

    @property
    def name(self):
        return self._name

    @property
    def graph(self):
        return self._graph

    def _validate_graph(self, x: TensorLike, name="x"):
        if isinstance(x, tf.RaggedTensor):
            self._validate_graph(x.flat_values, name=name)
        elif isinstance(x, tf.SparseTensor):
            self._validate_graph(x.indices, name=name)
        else:
            assert isinstance(x, tf.Tensor)
            if x.graph is not self._graph:
                raise ValueError("x is from a different graph - cannot add as input")

    def __enter__(self: T) -> T:
        ctx = self.graph.as_default()
        self._ctxs.append(ctx)
        ctx.__enter__()  # pylint:disable=no-member
        return self

    def __exit__(self, *args, **kwargs):
        ctx = self._ctxs.pop()
        ctx.__exit__(*args, **kwargs)

    def input(self, spec: TensorLikeSpec, name=None) -> TensorLike:
        if name is None:
            name = self.graph.unique_name(f"{self.name}/input")
        with self._graph.as_default():
            out = _spec_to_placeholder(spec, name=name)
            self._inputs.append(out)
        return out

    def input_like(self, x: TensorLike, name=None) -> TensorLike:
        if name is None:
            name = self.graph.unique_name(f"{self.name}/input")
        with self._graph.as_default():
            out = _placeholder_like(x, name=name)
        self._inputs.append(out)
        return out

    def batched_input_like(
        self, x: TensorLike, batch_size=None, name=None
    ) -> TensorLike:
        if name is None:
            name = self.graph.unique_name(f"{self.name}/input")
        with self._graph.as_default():
            out = _batched_placeholder_like(x, batch_size=batch_size, name=name)
            self._inputs.append(out)
        return out

    def add_output(self, x) -> None:
        self._validate_graph(x, "output")
        self._outputs.append(x)

    def build(
        self,
        inputs_structure=None,
        extra_outputs: Optional[Sequence[TensorLike]] = None,
    ) -> Optional[Callable]:
        inputs = self._inputs
        if inputs_structure is not None:
            inputs = tf.nest.pack_sequence_as(
                inputs_structure, inputs, expand_composites=True
            )

        outputs = self.outputs
        if extra_outputs is not None:
            for x in tf.nest.flatten(extra_outputs, expand_composites=True):
                self._validate_graph(x)
            outputs = (outputs, *extra_outputs)

        # If no outputs, return None. Maybe this should raise an error?
        if len(tf.nest.flatten(outputs, expand_composites=True)) == 0:
            return None

        return subgraph(self.graph.as_graph_def(add_shapes=True), inputs, outputs)

    @property
    def outputs(self) -> Tuple[TensorLike, ...]:
        return tuple(self._outputs)

    @property
    def inputs(self) -> Tuple[TensorLike, ...]:
        return tuple(self._inputs)

def _model_fn(
    model, inputs_structure, outputs_structure, squeezed,
):
    default_training = False  # required in tf 2.3

    def f(*inputs, training=None):
        if inputs_structure is not None:
            tf.nest.assert_same_structure(inputs, inputs_structure)
        inputs = tf.nest.flatten(inputs)
        assert len(inputs) == len(squeezed)
        inputs = [
            tf.expand_dims(inp, axis=0) if sq else inp
            for inp, sq in zip(inputs, squeezed)
        ]
        assert len(inputs) == len(model.inputs)
        outputs = model(inputs, training=training or default_training)
        if outputs_structure is not None:
            outputs = tf.nest.pack_sequence_as(outputs_structure, outputs)
        return outputs

    return f

class GraphModelBuilder(GraphBuilder):
    def __init__(self, *args, **kwargs):
        self._squeezed = []
        super().__init__(*args, **kwargs)

    def input(self, spec: TensorLikeSpec, name=None) -> TensorLike:
        with self._graph.as_default():
            if isinstance(spec, tf.RaggedTensorSpec):
                shape = spec._shape
                dtype = spec._dtype
            else:
                shape = spec.shape
                dtype = spec.dtype
            out = tf.keras.Input(
                shape=shape,
                batch_size=1,
                ragged=isinstance(spec, tf.RaggedTensorSpec),
                sparse=isinstance(spec, tf.SparseTensorSpec),
                dtype=dtype,
                name=name,
            )
            self._inputs.append(out)
            out = tf.squeeze(out, axis=0)
        self._squeezed.append(True)
        if isinstance(spec, tf.RaggedTensorSpec):
            assert isinstance(out, tf.RaggedTensor)
        elif isinstance(spec, tf.SparseTensorSpec):
            assert isinstance(out, tf.SparseTensor)
        else:
            assert isinstance(out, tf.Tensor)
        return out

    def input_like(self, x: TensorLike, name=None) -> TensorLike:
        with self._graph.as_default():
            out = tf.keras.Input(
                shape=x.shape,
                batch_size=1,
                ragged=isinstance(x, tf.RaggedTensor),
                sparse=isinstance(x, tf.SparseTensor),
                dtype=x.dtype,
                name=name,
            )
            self._inputs.append(out)
            out = tf.squeeze(out, axis=0)
        self._squeezed.append(True)
        assert type(x) is type(out)
        return out

    def batched_input_like(
        self, x: TensorLike, batch_size=None, name=None
    ) -> TensorLike:
        with self._graph.as_default():
            out = tf.keras.Input(
                shape=x.shape,
                batch_size=batch_size,
                ragged=isinstance(x, tf.RaggedTensor),
                sparse=isinstance(x, tf.SparseTensor),
                dtype=x.dtype,
                name=name,
            )
            self._inputs.append(out)
        self._squeezed.append(False)
        return out

    def build(
        self,
        inputs_structure=None,
        extra_outputs: Optional[Sequence[TensorLike]] = None,
    ):
        inputs = self._inputs
        outputs = self.outputs

        if extra_outputs is not None:
            for x in tf.nest.flatten(extra_outputs, expand_composites=True):
                self._validate_graph(x)
            outputs = (outputs, *extra_outputs)

        outputs_structure = outputs
        outputs = tf.nest.flatten(outputs)

        with self._graph.as_default():
            model = tf.keras.Model(inputs, outputs)
        if len(model.trainable_weights) != 0:
            raise RuntimeError("Model had trainable variables.")
        return _model_fn(
            model, inputs_structure, outputs_structure, squeezed=self._squeezed
        )

class ModelBuilder:
    """Similar to GraphModelBuilder, but without the graph."""

    def __init__(self, batch_size: Optional[int] = None):
        self._inputs = []
        self._outputs = []
        self._batch_size = batch_size

    def input(self, spec: TensorLikeSpec, name=None) -> TensorLike:
        out = _spec_to_input(spec, name=name)
        self._inputs.append(out)
        return out

    def input_like(self, x: TensorLike, name=None) -> TensorLike:
        out = tf.keras.Input(
            shape=x.shape[1:],
            batch_size=x.shape[0],
            ragged=isinstance(x, tf.RaggedTensor),
            sparse=isinstance(x, tf.SparseTensor),
            dtype=x.dtype,
            name=name,
        )
        self._inputs.append(out)
        return out

    def batched_input_like(self, x: TensorLike, name=None) -> TensorLike:
        out = tf.keras.Input(
            shape=x.shape,
            batch_size=self._batch_size,
            ragged=isinstance(x, tf.RaggedTensor),
            sparse=isinstance(x, tf.SparseTensor),
            dtype=x.dtype,
            name=name,
        )
        self._inputs.append(out)
        assert type(x) is type(out)
        return out

    def build(self, outputs) -> tf.keras.Model:
        if isinstance(outputs, (list, tuple)) and len(outputs) == 1:
            outputs = outputs[0]
        # no idea why this is necessary...
        # it works around some errors related to
        # https://github.com/tensorflow/tensorflow/issues/41239
        tf.keras.layers.Lambda(tf.identity)
        return tf.keras.Model(self._inputs, outputs)

    @property
    def outputs(self) -> Tuple[TensorLike, ...]:
        return tuple(self._outputs)

    @property
    def inputs(self) -> Tuple[TensorLike, ...]:
        return tuple(self._inputs)


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 31 19:48:29 2021

@author: jeanb
"""

import pandas as pd
import xlrd
from  matplotlib import pyplot as plt
import math
#import mglearn
#from sklearn.model_selection import train_test_split
#from sklearn.neighbors import KNeighborsRegressor
import numpy as np
from mpl_toolkits.mplot3d import Axes3D #to plot in 3D
import mpmath as mp

#------------------------------------------------------------------------------
                            #LECTURE DU FICHIER EXCEL

print("PARTIE 1 : Resprésentation graphique des données")
#Lecture du fichier excel
document = xlrd.open_workbook("Data_table_3.xlsx")
fichier = "Data_table_3.xlsx"
sheet = "Sheet1"
df = pd.read_excel(io=fichier, sheet_name=sheet)

#------------------------------------------------------------------------------
                            #CREATION D'UNE BDD OU S'ENTRAINER

ligne_debut = 1
ligne_fin = 6000
ligne = ligne_fin - ligne_debut

df_new = df[['id', 'referenceyear', 'tot_assets', 'capital', 'capital_surplus',
             'paidincapital', 'tot_deposits', 'cash', 'comm_portfolio',
             'securities', 'tot_credit']]

#df_data = df_new.head(ligne)
df_data = df_new[ligne_debut:ligne_fin]

print('------------------------------------')
print("Données à utiliser : \n{}".format(df_data))

df_new = df[['f1']]

#df_resultat = df_new.head(ligne)
df_resultat = df_new[ligne_debut:ligne_fin]

print('------------------------------------')
print("Faillit ou non des banques : \n{}".format(df_resultat))

#------------------------------------------------------------------------------
                            #CREATION DES DONNEES DE TRAINING ET DE TEST

#Choix du nombre de lignes qui vont etre sélectionnée -------------------------
calcul = 0.75*ligne
resultat = int(round(calcul))
#resultat = 75
fin = ligne_fin

#Initialisation des paramètres ------------------------------------------------
capital_ratio_1 = (df_data['capital'])/df_data['tot_assets']
df_data['capital_ratio'] = capital_ratio_1

liquid_ratio_1 = df_data['cash']/df_data['tot_assets']
df_data['liquidity_ratio'] = liquid_ratio_1

#Replace the nan values by 0
df_data.fillna(value=0, inplace = True)
df_resultat.fillna(value=0, inplace = True)

#Training data ----------------------------------------------------------------
Training =  np.array([df_data['referenceyear'][0:resultat],
                      df_data['capital_ratio'][0:resultat],
                      df_data['liquidity_ratio'][0:resultat]])

#plt.plot(Training[0], Training[1], label='Total assets')
plt.scatter(Training[0], Training[1], label='Capital Ratio')
plt.scatter(Training[0], Training[2], label='liquidity ratio')
#plt.plot(Training[0], Training[7], label='real_capital_ratio')
#plt.plot(Training[0], Training[8], label='balence commerciale')
plt.legend()

#Test data --------------------------------------------------------------------
Test = np.array([df_data['referenceyear'][resultat:fin],
                      df_data['capital_ratio'][resultat:fin],
                      df_data['liquidity_ratio'][resultat:fin]])

#Training matrice -------------------------------------------------------------
Xtrain = Training
Xtrain = Xtrain.T
Ytrain = np.array([df_resultat['f1'][0:resultat]])
Ytrain = Ytrain.T

#Test matrice -----------------------------------------------------------------
Xtest = Test
Xtest = Xtest.T
Ytest = np.array([df_resultat['f1'][resultat:fin]])
Ytest = Ytest.T

#Ajout de la colonne de 1 à la matrice Xtrain ---------------------------------
p = np.ones([len(Xtrain),1])
Xtrain = np.append(p, Xtrain, axis = 1)

#Ajout de la colonne de 1 à la matrice Xtest---------------------------------
p = np.ones([len(Xtest),1])
Xtest = np.append(p, Xtest, axis = 1)

#------------------------------------------------------------------------------
#--------------------------------DEFINITION FONCTIONS CFS----------------------

#Fonction hypothèse
def Hypothese(X, Theta):
    return X.dot(Theta)

#Fonction CFS
def CFS(X, Y):
    p1 = np.dot(X.T,X)
    p2 = np.dot(X.T, Y)
    #Theta = np.linalg.pinv(p1).dot(p2)
    Theta =  np.linalg.pinv(p1)
    Theta = Theta.dot(p2)
    #Theta = Theta[~np.isnan(Theta)]
    return Theta

#Fonction Erreur
def Erreur(x,y,theta):
    ypred = np.dot(x,theta)
    I = len(y)
    # boucle for pour parcourir les matrices
    for i in range (I):
        erreur = (1/I) * np.sum(np.square(ypred-y))  #MSE (mean square error)
    return erreur

#------------------------------------------------------------------------------
#------------------------------PARAMETRES OPTIMAUX-----------------------------

Theta_F = CFS(Xtrain, Ytrain)

print(Theta_F)

#Prédiction des données de training--------------------------------------------
Yprediction = Hypothese(Xtrain, Theta_F)
error = Erreur(Xtrain, Ytrain, Theta_F)
print("Erreur : ", error)

#Affichage des données---------------------------------------------------------
fig1 = plt.figure()
ax1 = fig1.gca(projection='3d')
ax1.scatter3D(Xtrain[:,2], Xtrain[:,3], Ytrain, color= 'green')
ax1.scatter3D(Xtrain[:,2], Xtrain[:,3], Yprediction, color= 'red')
ax1.set_xlabel('Capital Ratio', fontweight = 'bold')
ax1.set_ylabel('Liquidity ratio', fontweight = 'bold')
ax1.set_zlabel('Bankruptcy probability', fontweight = 'bold')
plt.show()

#☻Prediction des données de test-----------------------------------------------
#Yprediction2 = Hypothese(Xtest, Theta_F)
#error2 = Erreur(Xtest, Ytest, Theta_F)
#print("Erreur : ", error2)
#------------------------------------------------------------------------------

#Classe de regression logistique-----------------------------------------------
    #Appel permettant la résolution direct pour tout autre type de fichier

class LogisticRegression:

    #On instancie les paramètres propres à la régression
        #lr = alpha = précision
        #num_iter = nomre d'iterations !!ATENTION : Possible overfitting!!
        #fit intercept : ici il faut calculer le point d'interception
        #verbose : suite au traitement de beaucoup de données et de nombreuses itérations
   def __init__(self, lr = 0.01, num_iter = 100000, fit_intercept = True, verbose = False):
      self.lr = lr
      self.num_iter = num_iter
      self.fit_intercept = fit_intercept
      self.verbose = verbose

      #Ajout de la colonne de 1 à Xpar concatenate
   def __add_intercept(self, X):
      intercept = np.ones((X.shape[0], 1))
      return np.concatenate((intercept, X), axis=1)

    #Fonction Sigmoid
   def Sigmoid(self, z):
      return 1 / (1 + np.exp(-z))

    #Fonction erreur
   def Erreur(self, h, y):
      return (-y * np.log(h) - (1 - y) * np.log(1 - h)).mean()

    #Normalisation des données
   def fit(self, X, y):
      if self.fit_intercept:
        X = self.__add_intercept(X)
        self.theta = np.zeros(X.shape[1])
        for i in range(self.num_iter):
           z = np.dot(X, self.theta)
           h = self.Sigmoid(z)
           gradient = np.dot(X.T, (h - y)) / y.size #gradient : permet de faire des 'boucles' afin de trouver la valeur de Theta pour laquelle la fonction de coût est égale à 0
           self.theta -= self.lr * gradient

           z = np.dot(X, self.theta)
           h = self.Sigmoid(z)
           loss = self.Erreur(h, y)

           if(self.verbose ==True and i % 10000 == 0):
              print(f'loss: {loss} \t')

    #Retourne la valeur exacte de la valeur prédite
   def predict_prob(self, X):
       if self.fit_intercept:
           X = self.__add_intercept(X)
           return self.Sigmoid(np.dot(X, self.theta))

        #Arrondie la prédiction
   def predict(self, X):
      return self.predict_prob(X).round()

#Variables --------------------------------------------------------------------
model = LogisticRegression()
model = model.fit(Xtrain, Ytrain)
preds = model.predict(Xtrain)


from typing import Optional, Union  # noqa F401
import flask  # noqa F401

from pathlib import Path
import os

import pyjson5 as jsonc
import yaml

from hidebound.core.database import Database
from hidebound.server.api import API
import hidebound.core.config as hbc
import hidebound.core.tools as hbt
# ------------------------------------------------------------------------------

# inheriting from Singleton breaks init and init_app tests
class HideboundExtension:
    def __init__(self, app=None):
        # type: (Optional[flask.Flask]) -> None
        '''
        Initialize flask extension.

        Args:
            app (flask.Flask, optional): Flask app.
        '''
        if app is not None:
            self.init_app(app)

    def get_config(self, app):
        # type: (flask.Flask) -> dict
        '''
        Get config from envirnment variables or config file.

        Args:
            app (flask.Flask): Flask app.

        Returns:
            dict: Database config.
        '''
        config_path = os.environ.get('HIDEBOUND_CONFIG_FILEPATH', None)
        if config_path is not None:
            config = self._get_config_from_file(config_path)

        else:
            app.config.from_prefixed_env('HIDEBOUND')
            config = self._get_config_from_env(app)

        config = hbc.Config(config).to_native()
        return config

    def _get_config_from_file(self, filepath):
        # type: (Union[str, Path]) -> dict
        '''
        Get config from envirnment variables or config file.

        Args:
            filepath (str or Path): Filepath of hidebound config.

        Raises:
            FileNotFoundError: If HIDEBOUND_CONFIG_FILEPATH is set to a file
                that does not end in json, yml or yaml.

        Returns:
            dict: Database config.
        '''
        fp = Path(filepath)
        ext = fp.suffix.lower().lstrip('.')
        exts = ['json', 'yml', 'yaml']
        if ext not in exts:
            msg = 'Hidebound config files must end in one of these extensions: '
            msg += f'{exts}. Given file: {fp.as_posix()}.'
            raise FileNotFoundError(msg)

        with open(filepath) as f:
            if ext in ['yml', 'yaml']:
                return yaml.safe_load(f)
            return jsonc.load(f)

    def _get_config_from_env(self, app):
        # type: (flask.Flask) -> dict
        '''
        Get config from environment variables.

        Args:
            app (flask.Flask): Flask app.

        Returns:
            dict: Database config.
        '''
        dask = dict(
            cluster_type=app.config.get('DASK_CLUSTER_TYPE'),
            num_partitions=app.config.get('DASK_NUM_PARTITIONS'),
            local_num_workers=app.config.get('DASK_LOCAL_NUM_WORKERS'),
            local_threads_per_worker=app.config.get('DASK_LOCAL_THREADS_PER_WORKER'),
            local_multiprocessing=hbt.str_to_bool(
                app.config.get('DASK_LOCAL_MULTIPROCESSING', 'True')
            ),
            gateway_address=app.config.get('DASK_GATEWAY_ADDRESS'),
            gateway_proxy_address=app.config.get('DASK_GATEWAY_PROXY_ADDRESS'),
            gateway_public_address=app.config.get('DASK_GATEWAY_PUBLIC_ADDRESS'),
            gateway_auth_type=app.config.get('DASK_GATEWAY_AUTH_TYPE'),
            gateway_api_token=app.config.get('DASK_GATEWAY_API_TOKEN'),
            gateway_cluster_options=yaml.safe_load(
                str(app.config.get('DASK_GATEWAY_CLUSTER_OPTIONS', '[]'))
            ),
            gateway_shutdown_on_close=hbt.str_to_bool(
                app.config.get('DASK_GATEWAY_SHUTDOWN_ON_CLOSE', 'True')
            ),
        )
        return dict(
            ingress_directory=app.config.get('INGRESS_DIRECTORY'),
            staging_directory=app.config.get('STAGING_DIRECTORY'),
            include_regex=app.config.get('INCLUDE_REGEX', ''),
            exclude_regex=app.config.get('EXCLUDE_REGEX', r'\.DS_Store'),
            write_mode=app.config.get('WRITE_MODE', 'copy'),
            redact_regex=app.config.get('REDACT_REGEX', '(_key|_id|_token|url)$'),
            redact_hash=hbt.str_to_bool(app.config.get('REDACT_HASH', 'False')),
            specification_files=yaml.safe_load(
                str(app.config.get('SPECIFICATION_FILES', '[]'))
            ),
            workflow=yaml.safe_load(str(app.config.get(
                'WORKFLOW',
                '["delete", "update", "create", "export"]',
            ))),
            dask=dask,
            exporters=yaml.safe_load(str(app.config.get('EXPORTERS', '{}'))),
            webhooks=yaml.safe_load(str(app.config.get('WEBHOOKS', '[]'))),
        )

    def init_app(self, app):
        # type: (flask.Flask) -> None
        '''
        Add endpoints and error handlers to given app.

        Args:
            app (Flask): Flask app.
        '''
        app.extensions['hidebound'] = self
        app.register_blueprint(API)
        if not app.config['TESTING']:
            self.config = self.get_config(app)
            self.database = Database.from_config(self.config)


#!/usr/bin/env python

from __future__ import print_function

import subprocess
import sys

from setuptools import setup, find_packages
from codecs import open

from fbfeed import __version__

# Get the long description from the README file
with open("README.md", encoding="utf-8") as f:
    long_description = f.read()

# Bump version and generate CHANGELOG
if sys.argv[:2] == ["setup.py", "bump"]:
    subprocess.check_call("standard-version")  # npm install -g standard-version

# Tag and release the package to PyPI
if sys.argv[:2] == ["setup.py", "release"]:
    subprocess.check_call("git push")
    subprocess.check_call("git push --tags")
    subprocess.check_call("rm -rf dist/")
    subprocess.check_call("python setup.py sdist")
    subprocess.check_call("python setup.py bdist_wheel")
    subprocess.check_call("twine upload dist/*")
    sys.exit()

setup(
    name="mezzanine-facebook-feed",
    version=__version__,
    description="Connect Mezzanine sites to Facebook feeds",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/unplugstudio/mezzanine-facebook-feed",
    author="Unplug Studio",
    author_email="hello@unplug.studio",
    license="MIT",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Libraries",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
    ],
    keywords="django mezzanine facebook",
    packages=find_packages(),
    install_requires=["mezzanine>=4.3.0"],
    include_package_data=True,
)


import requests
from datetime import datetime, timezone, timedelta
from dateutil import parser
import logging

def get_jupyter_kernels(resource_url, user_token):
    """extracting information for each kernel on a Jupyter server"""
    sessions_url = f"{resource_url}api/sessions"
    headers = {"Authorization": f"token {user_token}"}
    response = requests.get(sessions_url, headers=headers)
    result = response.json()
    return result

def check_jupyter_needs_shutoff(kernels, idle_time_delta):
    """Given a Jupyter session check if any of the kernel's activity is more recent than idle limit.
    Return True if the server needs to be shut off. In the event there are no kernels, do not shut
    off the server, instead leave it to Saturn Cloud's built in logic to handle.
    """
    # getting time window to keep the server on
    min_time = datetime.now(timezone.utc) - idle_time_delta

    if len(kernels) == 0:
        return False  # There are no kernels at all so do not act

    for k in kernels:
        if (parser.parse(k["kernel"]["last_activity"]) >= min_time) or (
            k["kernel"]["execution_state"] != "idle"
        ):
            return False

    return True

def shutoff_resource(resource_id, base_url, user_token):
    """Turn off a Saturn Cloud resource for a user"""
    headers = {"Authorization": f"token {user_token}"}
    requests.post(
        f"{base_url}api/workspaces/{resource_id}/stop",
        headers=headers,
    )

time_delta_mapping = {
    "1 hour": timedelta(hours=1),  # hack for testing
    "6 hours": timedelta(hours=6),
    "24 hours": timedelta(hours=24),
    "3 days": timedelta(hours=24 * 3),
    "7 days": timedelta(hours=24 * 7),
}

def close_user_resources(base_url, username, user_token):
    """Given a user access token, close their idle resources
    idle_limit is the limit in hours before shutting down the resource.
    """

    # get all of their workspace resources
    workspaces = requests.get(
        f"{base_url}api/workspaces",
        headers={"Authorization": f"token {user_token}"},
    ).json()["workspaces"]

    # filter to only Jupyter servers (to exclude RStudio servers)
    jupyter_servers = list(filter(lambda w: w["resource_type"] == "Jupyter Workspace", workspaces))

    # for each resource check last activity of the Jupyter kernels
    for resource in jupyter_servers:
        if (
            resource["url"] is not None
        ):  # this means there is an actively running JupyterLab server we can query
            time_delta = time_delta_mapping.get(resource["auto_shutoff"], None)
            if time_delta is None:
                continue
            kernels = get_jupyter_kernels(resource["url"], user_token)
            needs_shutoff = check_jupyter_needs_shutoff(kernels, time_delta)
            if needs_shutoff:
                logging.warning(f"Shutting down resource: {username}/{resource['name']}")
                shutoff_resource(resource, base_url, user_token)


import pymongo
import twitterAPI

MONGODB_URL = ''

def add_account():
    name = input("Enter a twitter name: ")
    connect = pymongo.MongoClient(MONGODB_URL)
    db = connect["EC601_MiniProject3_db"]
    myset = db["project1_record"]
    if myset.find({'username': name}):
        print("This account already existed")
        return

    time_offset = twitterAPI.main(name, 40)

    account = {"username": name, "labels": time_offset}
    myset.insert_one(account)

def query_account():
    account = input("Enter an existed twitter name: ")
    connect = pymongo.MongoClient(MONGODB_URL)
    database = connect["EC601_MiniProject3_db"]
    myset = database["project1_record"]
    match = myset.find({'username': account})
    if not match:
        print("No match found!")
    else:
        for row in match:
            print("account =", row['username'], "\nlabels=", row['labels'])

def main():
    do1 = input('Do you want to add an account? (y/n): ')
    if do1 not in ['y', 'n']:
        print('Invalid input')
    elif do1 in ['y']:
        add_account()
    else:
        do2 = input('Do you want to query an existed account? (y/n): ')
        if do2 not in ['y', 'n']:
            print('Invalid input')
        elif do2 in ['y']:
            query_account()
        else:
            pass

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

#########################################################################################################################
# IMPORTATION
#########################################################################################################################
import sys
import PyQt5
from PyQt5.QtWidgets import QWidget, QFrame, QApplication
from PyQt5.QtWidgets import QVBoxLayout,QHBoxLayout,QSplitter,QGridLayout
from PyQt5.QtWidgets import QLabel, QPushButton, QLineEdit, QSpinBox, QDoubleSpinBox, QSlider, QComboBox, QFileDialog
from PyQt5.QtCore    import Qt, QThread, QTimer, QObject, pyqtSignal, pyqtSlot, QRect, QLine
from PyQt5.QtGui     import QPainter

import numpy     as np
import pyqtgraph as pg
import os

from s_LogDisplay_class               import LogDisplay
from s_Workers_class                  import *
from s_ToolObjects_class              import QVLine
from s_Miscellaneous_functions        import *
from s_SimuCamera_class               import SimuCamera
from s_Camera_class                   import Camera

import time
#########################################################################################################################
# FUNCTIONS
#########################################################################################################################

class CameraDisplay(QWidget):
    frame_updated = pyqtSignal()
    # ~~~~~~~~~~~~~~~~~~~~~~~~ #
    def __init__(self, camera=None, log=None, fps=10.):
        super().__init__()
        # ---  --- #
        if log != None:
            self.log = log
        else:
            self.log = LogDisplay()
        #self.log.show()
        # --- default --- #
        self.fps       = fps
        self.normalise_hist = True
        self.cmap      = 'jet'
        # --- main attriute --- #
        self.camera    = camera
        self.contview  = ContinuousView(fps=self.fps)
        self.timer     = pg.QtCore.QTimer() #QTimer()# pg.QtCore.QTimer()
        self.qlabl_max = QLabel()
        self.isOn      = False
        self.frame     = None
        # --- color acuisition --- #
        self.initColorDic()
        # ---  --- #
        self.initUI()
        self.initCamera(camera)
        # ---  --- #
        self.camera.setExposure( self.exposure.value() )

    def initUI(self):
        # ---  --- #
        self.layout      = QVBoxLayout(self)
        self.initView()
        # --- button widget --- #
        self.button_startstop = QPushButton('Start/Stop')
        self.button_startstop.setStyleSheet("background-color: red")
        self.button_nextFrame = QPushButton('Next frame')
        # ---  --- #
        self.fps_input     = QSpinBox()
        self.fps_input.setRange(1, 48)
        self.fps_input.setValue(self.fps)
        self.exposure      = QDoubleSpinBox()
        self.exposure.setSingleStep(0.01)
        self.cam_framerate = QDoubleSpinBox()
        self.exposure.setSingleStep(0.01)
        self.pixelclock    = QDoubleSpinBox()
        self.which_camera  = QComboBox()
        self.which_camera.addItem('USB camera')
        self.which_camera.addItem('From Image Dir.')
        # --- set default --- #
        self.exposure.setRange(0.10, 99.0)
        self.exposure.setValue(12.5)
        self.cam_framerate.setRange(1.00, 15.0)
        self.cam_framerate.setValue( 10.0 )
        self.pixelclock.setRange(5, 30)
        self.pixelclock.setValue(20)
        # --- connections --- #
        self.button_startstop.clicked.connect(self.startStop_continuous_view)
        self.fps_input.valueChanged.connect(self.setFPS)
        self.button_nextFrame.clicked.connect( self.nextFrame )
        self.exposure.valueChanged.connect( self.update_exposure )
        self.cam_framerate.valueChanged.connect( self.update_camFPS )
        self.pixelclock.valueChanged.connect( self.update_pixelClock )
        self.which_camera.currentIndexChanged.connect( self.changeCameraStyle )
        # --- layout --- #
        label_1 = QLabel('fps :')
        label_1.setWordWrap(True)
        label_2 = QLabel('value max:')
        label_2.setWordWrap(True)
        label_3 = QLabel('Which camera')
        label_3.setWordWrap(True)
        label_4 = QLabel('Exposure (ms):')
        label_4.setWordWrap(True)
        label_5 = QLabel('Camera fps:')
        label_5.setWordWrap(True)
        label_6 = QLabel('Pixel clock (MHz):')
        label_6.setWordWrap(True)
        grid = QGridLayout()
        grid.addWidget( self.button_startstop, 0,0)
        grid.addWidget( self.button_nextFrame, 0,1)
        grid.addWidget( label_1              , 0,2)
        grid.addWidget( self.fps_input       , 0,3)
        grid.addWidget( label_2              , 0,4)
        grid.addWidget( self.qlabl_max       , 0,5)
        grid.addWidget(QVLine()              , 0,6 , 2,1)
        grid.addWidget(label_3               , 0,7)
        grid.addWidget( self.which_camera    , 1,7 , 2,1)
        grid.addWidget(label_4               , 1,0)
        grid.addWidget( self.exposure        , 1,1)
        grid.addWidget(label_5               , 1,2)
        grid.addWidget( self.cam_framerate   , 1,3)
        grid.addWidget(label_6               , 1,4)
        grid.addWidget( self.pixelclock      , 1,5)
        self.layout.addLayout(grid)
        self.layout.addWidget(self.image_view)
        self.setLayout(self.layout)

    def initColorDic(self):
        self.colordic = {}
        # --- jet-like cmap --- #
        alpha     = 1.0
        positions = [0.2, 0.5, 0.75, 1.0]
        colors    = [[0,0,1.,alpha],[0,1.,1.,alpha],[1.,1.,0,alpha],[170/255,0,0,alpha]] #colors    = ['#0000ff', '#00ffff', '#ffff00', '#aa0000']
        self.colordic['jet'] = pg.ColorMap(positions, colors)
        # --- jet reversed cmap --- #
        positions_r = [1-p_ for p_ in positions]
        self.colordic['jet_r'] = pg.ColorMap(positions_r, colors)
        # --- plasma cmap --- #
        self.colordic['plasma'] = generatePgColormap('plasma')

    def initView(self):
        self.image_view     = pg.ImageView()
        # ---  --- #
        self.image_view.setColorMap(self.colordic[self.cmap])
        self.image_view.setLevels(0,255)
        self.image_view.getHistogramWidget().item.setHistogramRange(0,255)
        # ---  --- #
        self.image_view.setMinimumWidth(800)
        self.image_view.setMinimumHeight(600)

    def initCamera(self, camera):
        self.default_camera = camera
        self.simu_camera    = SimuCamera(directory_path=os.getcwd(), log=self.log)
        # ---  --- #
        self.camera = self.default_camera
        if not self.camera.isCameraInit:
            self.camera.__init__()

    def hideHistogram(self):
        self.image_view.ui.histogram.hide()

    def hidePlotButtons(self):
        self.image_view.ui.roiBtn.hide()
        self.image_view.ui.menuBtn.hide()

    def setFPS(self):
        self.fps = self.fps_input.value()
        self.contview.setFPS( self.fps )
        self.timer.setInterval(1e3/self.fps)

    def update_frame(self):
        self.frame = self.camera.get_frame()
        self.qlabl_max.setText( str(np.max(self.frame)) )
        self.image_view.setImage(self.frame.T, autoHistogramRange=False, autoLevels=False)
        self.frame_updated.emit()

    def update_exposure(self):
        self.camera.setExposure( self.exposure.value() )
        self.log.addText( 'New exposure: {}ms'.format( self.camera.getExposure() ) )
        # ---  --- #
        new_pixelclock = self.camera.getPixelClock()
        self.pixelclock.setValue( new_pixelclock )
        new_camFPS     = self.camera.getFrameRate()
        self.cam_framerate.setValue( new_camFPS )

    def update_camFPS(self):
        newfps = self.camera.setFrameRate( self.cam_framerate.value() )
        self.log.addText('New camera fps: {:.3f}'.format( self.camera.getFrameRate() ) )
        self.cam_framerate.setValue(newfps)
        # ---  --- #
        new_pixelclock = self.camera.getPixelClock()
        self.pixelclock.setValue( new_pixelclock )
        new_exposure   = self.camera.getExposure()
        self.exposure.setValue( new_exposure )

    def update_pixelClock(self):
        self.camera.setPixelClock( int(self.pixelclock.value()) )
        self.log.addText( 'New pixel clock: {}Mhz'.format( self.camera.getPixelClock() ))
        # ---  --- #
        new_camFPS     = self.camera.getFrameRate()
        self.cam_framerate.setValue( new_camFPS )
        new_exposure   = self.camera.getExposure()
        self.exposure.setValue( new_exposure )

    def nextFrame(self):
        wasOn = self.isOn
        if not self.isOn:
            self.camera.capture_video()
        # ---  --- #
        self.update_frame()
        # ---  --- #
        if not wasOn:
            self.camera.capture_video()

    def startStop_continuous_view(self):
        if  self.isOn:
            self.stop_continuous_view()
            self.button_startstop.setStyleSheet("background-color: red")
            self.button_nextFrame.setFlat(False)
            self.button_nextFrame.setEnabled(True)
        else:
            self.start_continuous_view()
            self.button_startstop.setStyleSheet("background-color: green")
            self.button_nextFrame.setFlat(True)
            self.button_nextFrame.setEnabled(False)

    def start_continuous_view(self):
        self.camera.capture_video()
        if   True:
            self.start_continuous_view_qtimer()
        elif False:
            self.start_continuous_view_qthread()
        # ---  --- #
        self.isOn = True

    def stop_continuous_view(self):
        self.camera.stop_video()
        if   True:
            self.stop_continuous_view_qtimer()
        elif False:
            self.stop_continuous_view_qthread()
        # ---  --- #
        self.isOn = False

    def start_continuous_view_qthread(self):
        # ---  --- #
        self.thread = QThread()
        self.thread.setTerminationEnabled(True)
        # --- connect --- #
        self.contview.moveToThread(self.thread)
        self.thread.started.connect(self.contview.startFeed)
        self.contview.newshot.connect(self.update_image)
        self.contview.finished.connect(self.thread.quit)
        # ---  --- #
        self.thread.start()

    def stop_continuous_view_qthread(self):
        self.contview.stopFeed()

    def start_continuous_view_qtimer(self):
        # ---  --- #
        self.timer.timeout.connect(self.update_frame)
        self.timer.start(1e3/self.fps) #ms

    def stop_continuous_view_qtimer(self):
        self.timer.stop()

    def changeCameraStyle(self):
        self.camera.stop_video()
        # ---  --- #
        indx = self.which_camera.currentIndex()
        if   indx == 0:
            self.camera = self.default_camera
            # ---  --- #
            self.exposure.setEnabled(True)
            self.cam_framerate.setEnabled(True)
            self.pixelclock.setEnabled(True)
        elif indx == 1:
            self.camera.stop_video()
            dir_path    = QFileDialog().getExistingDirectory()
            self.log.addText('NEW DIR PATH: {}'.format(dir_path))
            self.simu_camera.setDirectoryPath( dir_path )
            self.simu_camera.initialize()
            self.camera = self.simu_camera
            # ---  --- #
            self.exposure.setEnabled(False)
            self.cam_framerate.setEnabled(False)
            self.pixelclock.setEnabled(False)

#########################################################################################################################
# CODE
#########################################################################################################################
if __name__ == '__main__':
    print('STARTING')
    dir_path = '/home/cgou/ENS/STAGE/M2--stage/Camera_acquisition/Miscellaneous/Camera_views/'
    camera   = Camera(cam_id=0)
    if not camera.isCameraInit:
        camera = SimuCamera(0, directory_path=dir_path)
        camera.__info__()

    app = QApplication([])
    start_window = CameraDisplay(camera)
    start_window.show()
    app.exit(app.exec_())
    print('Close camera')
    camera.close_camera()

    print('FINISHED')


import os
import cv2
import pickle

class faces():

    def __init__(self):
        f = os.path.dirname(cv2.__file__) +  '/data/haarcascade_frontalface_alt2.xml'
        # print("* CascadeClassifier:", f)
        self.face_cascade = cv2.CascadeClassifier(f)

        self.recognizer = cv2.face.LBPHFaceRecognizer_create()
        self.recognizer.read("./trainer.yml")

        self.labels = {"person_name": 1}
        with open("labels.pickle", 'rb') as f:
            og_labels = pickle.load(f)
            self.labels = {v:k for k,v in og_labels.items()}

    def find(self, frame):

        gray  = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        face_list = self.face_cascade.detectMultiScale(gray, scaleFactor=1.5, minNeighbors=5)

        for (x, y, w, h) in face_list:
            #print(x,y,w,h)
            roi_gray = gray[y:y+h, x:x+w]
            roi_color = frame[y:y+h, x:x+w]

            id_, conf = self.recognizer.predict(roi_gray)

            # print(conf, labels[id_])
            if conf >= 50 and conf <= 100:
                font = cv2.FONT_HERSHEY_DUPLEX
                name = self.labels[id_]

                color = (255, 0, 0)  #BGR
                stroke = 1
                end_cord_x = x + w
                end_cord_y = y + h
                cv2.rectangle(frame, (x, y), (end_cord_x, end_cord_y), color, stroke )
                cv2.rectangle(frame, (x, end_cord_y + 20 ), (end_cord_x, end_cord_y), color, cv2.FILLED )
                cv2.putText(frame, name, (x + 6, end_cord_y + 16), font, 0.5, (255,255,255), stroke, cv2.LINE_AA)

        return frame


# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib.rnn import LSTMCell
import time
from utils import get_logger
from data_helper import batch_iter
import numpy as np
from utils import focal_loss

class BiLSTM_Model(object):
    def __init__(self, args, tag2label, model_w2v, paths, config):
        self.batch_size = args['batch_size']
        self.epoch_num = args['epoch']
        self.hidden_dim = args['hidden_dim']
        self.embedding_dim = args['embedding_dim']
        self.dropout_keep_prob = args['dropout']
        self.lr_ = args['lr']
        # 学习率自动衰减
        self.global_step = tf.Variable(0)
        self.lr = tf.train.exponential_decay(self.lr_, self.global_step, decay_steps=4,
                                             decay_rate=0.98, staircase=False)
        # 词向量矩阵，添加一行全0向量，对应没出现过的词<UNK>
        self.embeddings = np.r_[model_w2v.wv.vectors, [[0] * self.embedding_dim]]
        self.tag2label = tag2label
        self.num_tags = len(tag2label)
        self.shuffle = args['shuffle']
        self.model_path = paths['model_path']
        self.summary_path = paths['summary_path']
        self.logger = get_logger(paths['log_path'])
        self.result_path = paths['result_path']
        self.config = config

    def build_graph(self):
        """
        构建TensorFlow网络结构
        可以添加attention层去优化
        :return:
        """
        self.add_placeholders()
        self.lookup_layer_op()
        self.biLSTM_layer_op()
        self.accuracy_op()
        self.loss_op()
        self.trainstep_op()
        self.init_op()

    def add_placeholders(self):
        """
        使用tf的占位符，初始化3个属性
        :return:
        """
        self.word2ids = tf.placeholder(tf.int32, shape=[None, None], name="word2ids")
        self.labels = tf.placeholder(tf.int32, shape=[None, None], name="labels")
        self.dropout_pl = tf.placeholder(dtype=tf.float32, shape=[], name="dropout")

    def lookup_layer_op(self):
        """
        在预训练的词向量矩阵中寻找max_len个词向量，一个word_id对应一个词向量，id数即为词向量矩阵的第几行
        预训练词向量矩阵是一个可优化的点
        """
        with tf.variable_scope("words"):
            _word_embeddings = tf.Variable(self.embeddings,
                                           dtype=tf.float32,
                                           name="_word_embeddings")
            word_embeddings = tf.nn.embedding_lookup(params=_word_embeddings,
                                                     ids=self.word2ids,
                                                     name="word_embeddings")
        self.word_embeddings = tf.nn.dropout(word_embeddings, self.dropout_pl)

    def biLSTM_layer_op(self):
        with tf.variable_scope("bi-lstm"):
            # 前向lstm
            cell_fw = LSTMCell(self.hidden_dim)
            # 反向lstm
            cell_bw = LSTMCell(self.hidden_dim)
            (output_fw_seq, output_bw_seq), _ = tf.nn.bidirectional_dynamic_rnn(
                cell_fw=cell_fw,
                cell_bw=cell_bw,
                inputs=self.word_embeddings,
                dtype=tf.float32)

            output = tf.concat([output_fw_seq, output_bw_seq], axis=-1)

            # 将词向量取平均，维度变化：[batch_size, max_len, hidden_dim] -> [batch_size, hidden_dim]
            # 即将max_len个hidden_dim维的词向量合并为一个
            output = tf.reduce_sum(output, axis=1)
            output = tf.reshape(output, [-1, 2 * self.hidden_dim])
            output = tf.nn.dropout(output, self.dropout_pl)

        with tf.variable_scope("proj"):
            W = tf.get_variable(name="W",
                                shape=[2 * self.hidden_dim, self.num_tags],
                                initializer=tf.contrib.layers.xavier_initializer(),
                                dtype=tf.float32)

            b = tf.get_variable(name="b",
                                shape=[self.num_tags],
                                initializer=tf.zeros_initializer(),
                                dtype=tf.float32)

            pred = tf.matmul(output, W) + b
            self.logits = tf.nn.dropout(pred, self.dropout_pl)

    def loss_op(self):
        """
        损失函数，可尝试替换其他损失函数去优化，如focal loss
        """
        losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.labels)
        self.loss = tf.reduce_mean(losses)
        tf.summary.scalar("loss", self.loss)

    def accuracy_op(self):
        self.prediction = tf.cast(tf.argmax(self.logits, axis=1), tf.int32)
        correct_prediction = tf.equal(self.prediction, tf.cast(tf.argmax(self.labels, axis=1), tf.int32))
        self.correct_num = tf.reduce_sum(tf.cast(correct_prediction, tf.float32))
        self.acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name="acc")

    def trainstep_op(self):
        with tf.variable_scope("train_step"):
            self.global_step = tf.Variable(0, name="global_step", trainable=False)
            optim = tf.train.AdamOptimizer(learning_rate=self.lr)
            self.train_op = optim.minimize(self.loss)

    def init_op(self):
        """变量初始化"""
        self.init_op = tf.global_variables_initializer()

    def add_summary(self, sess):
        """
        打印概览
        :param sess:
        :return:
        """
        self.merged = tf.summary.merge_all()
        self.file_writer = tf.summary.FileWriter(self.summary_path, sess.graph)

    def train(self, train, dev):
        """
        训练入口
        :param train:训练集
        :param dev: 验证集
        :return:
        """
        saver = tf.train.Saver(tf.global_variables())

        with tf.Session(config=self.config) as sess:
            sess.run(self.init_op)
            self.add_summary(sess)

            for epoch in range(self.epoch_num):
                self.run_one_epoch(sess, train, dev, epoch, saver)

    def test(self, test):
        """
        测试入口
        :param test:测试集
        :return:
        """
        saver = tf.train.Saver()
        with tf.Session(config=self.config) as sess:
            self.logger.info('=========== testing ===========')
            saver.restore(sess, self.model_path)
            self.dev_one_epoch(sess=sess, dev=test)

    def demo(self, demo_sent):
        """
        测试一条文本，返回值可根据需求更改
        """
        saver = tf.train.Saver()
        with tf.Session(config=self.config) as sess:
            saver.restore(sess, self.model_path)
            y_pred = self.predict_one(sess, demo_sent)[0]
            label = self.tag2label[str(y_pred)]
            return label

    def run_one_epoch(self, sess, train, dev, epoch, saver):
        """
        训练1个opoch
        :param sess:session
        :param train:训练集
        :param dev:验证集
        :param epoch:当前epoch
        :param saver:tf模型保存对象
        :return:
        """
        num_batches = (len(train[0]) + self.batch_size - 1) // self.batch_size
        start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        batches = batch_iter(train, self.batch_size)
        avg_loss = 0
        avg_acc = 0
        for step, (seqs, labels) in enumerate(batches):
            step_num = epoch * num_batches + step + 1
            feed_dict = self.get_feed_dict(seqs, labels, self.dropout_keep_prob)
            _, loss_train, acc_train, summary, step_num_ = sess.run(
                [self.train_op, self.loss, self.acc, self.merged, self.global_step],
                feed_dict=feed_dict)
            avg_loss += loss_train / num_batches
            avg_acc += acc_train / num_batches
            self.file_writer.add_summary(summary, step_num)
        self.logger.info(
            '{} epoch {}, loss: {:.4}, acc: {:.4}'.format(start_time, epoch + 1, avg_loss, avg_acc))
        saver.save(sess, self.model_path, global_step=epoch + 1)

        self.logger.info('===========validation / test===========')
        self.dev_one_epoch(sess, dev, epoch)

    def get_feed_dict(self, seqs, labels=None, dropout=None):
        """
        获取tf网络模型需要“喂”的数据
        :param seqs: 文本
        :param labels:
        :param dropout:
        :return: feed_dict
        """
        feed_dict = {self.word2ids: seqs}
        if labels is not None:
            feed_dict[self.labels] = labels
        if dropout is not None:
            feed_dict[self.dropout_pl] = dropout

        return feed_dict

    def dev_one_epoch(self, sess, dev, epoch=None):
        """
        :param sess:
        :param dev:
        :param epoch:
        :return:
        """
        num_batches = (len(dev[0]) + self.batch_size - 1) // self.batch_size
        avg_acc = 0
        for seqs, labels in batch_iter(dev, self.batch_size):
            feed_dict = self.get_feed_dict(seqs, labels, 1.0)
            acc = sess.run(self.acc, feed_dict=feed_dict)
            avg_acc += acc / num_batches
        if epoch is not None:
            self.logger.info('epoch {}, dev acc : {:.4}'.format(epoch + 1, avg_acc))
        else:
            self.logger.info('Test acc : {:.4}'.format(avg_acc))

    def predict_one(self, sess, seqs):
        """
        预测一条
        :param sess:
        :param seqs:
        :return y_pred: 预测标签对应的id
        """
        seqs = np.array([seqs])
        feed_dict = self.get_feed_dict(seqs=seqs, dropout=1.0)
        y_pred = sess.run(self.prediction, feed_dict=feed_dict)

        return y_pred


#!/bin/python3

import json
import requests
import logging
from datetime import datetime, timedelta

def configure_logging():
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO)

def _set_headers(username: str = None, api_key: str = None):
    headers = {'Content-Type': 'application/json'}
    if username: headers['Authorization'] = f'Basic {username}'
    if api_key: headers['PRIVATE-TOKEN'] = f'{api_key}'
    return headers

def _set_uri():
    return '{gitlab}/api/v4'

def get_access_token():
    return ''

def process_api_request(url: str, verb: str, headers: dict, data: dict = None, params: dict = None):
    try:
        if data: r = getattr(requests, verb.lower())(url,headers=headers,data=json.dumps(data))
        elif params: r = getattr(requests, verb.lower())(url,headers=headers,params=params)
        else: r = getattr(requests, verb.lower())(url,headers=headers)

        r.raise_for_status()
    except Exception as e:
        logging.error(f'An error occured executing the API call: {e}')

    try:
        return r.json()
    except Exception as e:
        logging.error(f'An error occured loading the content: {e}')
        return None

def get_jobs(token, page, project_id):
    data = {
        'page' : page,
        'per_page' : 50
    }
    # List project jobs | https://docs.gitlab.com/ee/api/jobs.html#list-project-jobs
    return process_api_request(f'https://{_set_uri()}/projects/{project_id}/jobs', 'GET', _set_headers(api_key=token), params=data)

def get_job_count(token, page, project_id):
    count = get_jobs(token, page, project_id)
    if len(count) > 0: return True
    return False

def delete_artifacts(token, project_id, job_id):
    # Delete job artifacts | https://docs.gitlab.com/ee/api/job_artifacts.html#delete-artifacts
    try:
        process_api_request(f'https://{_set_uri()}/projects/{project_id}/jobs/{job_id}/artifacts', 'DELETE', _set_headers(api_key=token))
    except Exception as e:
        logging.error(f'An error occured loading the content: {e}')
    return False

def main():
    configure_logging()
    token=get_access_token()
    project_id='254'

    logging.info(f"Start cleaning project: {project_id}")

    page_count = 1
    while get_job_count(token, page_count, project_id):
        jobs = get_jobs(token, page_count, project_id)

        for entry in jobs:
            create_date = datetime.strptime(entry["created_at"], '%Y-%m-%dT%H:%M:%S.%fZ')
            time_between_insertion = datetime.now() - create_date

            if time_between_insertion.days>=30 and len(entry["artifacts"]) > 0:
                #logging.info(f"Remove job artifacts for job {entry['id']}")
                delete_artifacts(token,project_id,entry["id"])
        page_count +=1

    logging.info(f"Finished cleaning project {project_id}")

if __name__ == "__main__":
    main()


#coding:utf-8

import http.server

port = 80 #par defaut pour le port http
adress = ("", port) #comme c'est en local pas besoin de mettre le localhost

server = http.server.HTTPServer #ici on demarre le server

handler = http.server.CGIHTTPRequestHandler #un gestionnaire pour le http,mais foctionnel pour le script cgi
handler.cgi_directories = ["/"] #là ou il va execute les script on le met par defaut sur le meme ficheir

httpd = server(adress, handler) #ici on crée le server

print(f"server demarre sur le port {port}")

httpd.serve_forever()


from dataclasses import dataclass

from huobi.rest.endpoints import DONT_SEND, Endpoint
from huobi.rest.enums import HttpMethod

@dataclass
class LoanInterestRateAndQuotaIsolatedEndpoint(Endpoint):
    name: str = 'MarginLoanCI/LoanInterestRateAndQuotaIsolated'
    raw_path: str = '/v1/margin/loan-info'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'symbols': DONT_SEND,
    }

@dataclass
class SearchPastMarginOrdersIsolatedEndpoint(Endpoint):
    name: str = 'MarginLoanCI/SearchPastMarginOrders'
    raw_path: str = '/v1/margin/loan-orders'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'symbol': DONT_SEND,
        'states': DONT_SEND,
        'start-date': DONT_SEND,
        'end-date': DONT_SEND,
        'from': DONT_SEND,
        'direct': DONT_SEND,
        'size': DONT_SEND,
        'sub-uid': DONT_SEND,
    }

@dataclass
class BalanceOfTheMarginLoanAccountEndpoint(Endpoint):
    name: str = 'MarginLoanCI/BalanceOfTheMarginLoanAccount'
    raw_path: str = '/v1/cross-margin/accounts/balance'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'sub-uid': DONT_SEND,
    }

@dataclass
class BalanceOfTheMarginLoanAccountIsolatedEndpoint(Endpoint):
    name: str = 'MarginLoanCI/BalanceOfTheMarginLoanAccountIsolated'
    raw_path: str = '/v1/margin/accounts/balance'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'symbol': DONT_SEND,
        'sub-uid': DONT_SEND,
    }

@dataclass
class LoanInterestRateAndQuotaCrossEndpoint(Endpoint):
    name: str = 'MarginLoanCI/LoanInterestRateAndQuotaCross'
    raw_path: str = '/v1/cross-margin/loan-info'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'symbol': DONT_SEND,
        'sub-uid': DONT_SEND,
    }

@dataclass
class SearchPastMarginOrdersCrossEndpoint(Endpoint):
    name: str = 'MarginLoanCI/SearchPastMarginOrders'
    raw_path: str = '/v1/cross-margin/loan-orders'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'start-date': DONT_SEND,
        'end-date': DONT_SEND,
        'currency': DONT_SEND,
        'state': DONT_SEND,
        'from': DONT_SEND,
        'direct': DONT_SEND,
        'size': DONT_SEND,
        'sub-uid': DONT_SEND,
    }

@dataclass
class RepaymentRecordReferenceEndpoint(Endpoint):
    name: str = 'MarginLoanCI/RepaymentRecordReference'
    raw_path: str = '/v2/account/repayment'
    method: HttpMethod = HttpMethod.GET
    query_params = {
        'repayId': DONT_SEND,
        'accountId': DONT_SEND,
        'currency': DONT_SEND,
        'startTime': DONT_SEND,
        'endTime': DONT_SEND,
        'sort': DONT_SEND,
        'limit': DONT_SEND,
        'fromId': DONT_SEND,
    }


from red_contour_grip import RedContoursPipeline

import config

import cv2
import time
import math
import numpy

def calculate_coords(u, v):

    # pulled from https://github.com/ligerbots/VisionServer, BallFinder2020

    cam_matrix = config.cameramtx

    x_prime = (u - cam_matrix[0, 2]) / cam_matrix[0, 0]
    y_prime = -(v - cam_matrix[1, 2]) / cam_matrix[1, 1]

    ax = math.atan2(x_prime, 1.0)
    ay = math.atan2(y_prime * math.cos(ax), 1.0)

    print(str(ax) + " vs " + str(math.atan(x_prime)))

    target_height = 21
    camera_height = 4
    tilt_angle = 0

    d = (target_height - camera_height) / math.tan(tilt_angle + ay)

    return math.degrees(ax), math.degrees(ay), d

def calc_contours(c):
    M = cv2.moments(c)
    if M['m00'] != 0:
        cx = int(M['m10']/M['m00'])
        cy = int(M['m01']/M['m00'])

    return cx,cy

cap = cv2.VideoCapture(1)

# Run the pipeline on the video stream
while True:

    _, frame = cap.read()

    if frame is None:
        continue

    # Undistort the frame
    temp = cv2.undistort(frame, config.cameramtx, config.dist, None, config.cameramtx)
    # TODO Crop the frame using roi
    frame = temp

    # Process frame
    visionPipeline = RedContoursPipeline() # LemonVisionGripPipeline()
    visionPipeline.process(frame)

    # Retrieve the blobs from the pipeline
    contours = visionPipeline.filter_contours_output # tuple of KeyPoint objects

    print(str(len(contours)) + " blobs detected")

    if len(contours) > 0:
        # Find largest contour
        largest_contour = []
        largest_contour_idx = 0
        for i in range(len(contours)):
            if (len(contours[i]) > len(largest_contour)):
                largest_contour = contours[i]
                largest_contour_idx = i

        # Find centroid of contour
        x, y = calc_contours(largest_contour)
        x = int(x)
        y = int(y)

        cv2.line(frame, (x - config.center_line_length, y), (x + config.center_line_length, y), config.color, config.line_thickness)
        cv2.line(frame, (x, y - config.center_line_length), (x, y + config.center_line_length), config.color, config.line_thickness)

        temp = cv2.drawContours(frame, contours, largest_contour_idx, config.color, config.line_thickness, cv2.LINE_8, maxLevel=0)
        frame = temp

        # Calculate angle to target
        pitch_angle, yaw_angle, d = calculate_coords(x, y)

        temp = cv2.putText(frame, "(" + str(round(pitch_angle, 3)) + ", " + str(round(yaw_angle, 3)) + ", " + str(round(d, 3)) +  ")", (x + config.text_offset, y - config.text_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, config.color, config.line_thickness)
        frame = temp

        print("x: " + str(x) + "; y: " + str(y))
        print("pitch: " + str(pitch_angle) + "; yaw: " + str(yaw_angle))
        print(config.cameramtx[0, 2])
        print(config.cameramtx[0][2])

    cv2.imshow('image', frame)
    cv2.imshow('mask', visionPipeline.hsv_threshold_output)

    if cv2.waitKey(1000) & 0xFF == ord('q'):
        break


# -*- coding: utf-8 -*-
# To install 'inotify'
# $ pip install inotify --no-binary :all:
#
# To install 'boto3'
# $ pip install boto3
#
# To connect AWS, you should set up authentication credentials.
# note: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html
#
#
# This is Amazon S3 Examples
# > https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-examples.html

import inotify.adapters
import boto3

# Create an S3 client
s3 = boto3.client('s3')

i = inotify.adapters.Inotify()

# monitor specific folders
# 기본 dir 에 python 파일이 존재하고, jetson 에 sd card 에서 detect 한 이미지가 업로드되는 상황
watch_path = '../../media/nvidia/test_sd'
i.add_watch(watch_path)

for event in i.event_gen(yield_nones = False):
    (header, type_names, path, saved_filename) = event
    # print("PATH=[{}]\tFILENAME=[{}]\tEVENT_TYPES={}".format(path, filename, type_names))

    # 모니터 하고 있는 폴더에 파일이 생성될 때만 실행
    if type_names[0] == 'IN_CREATE':

        # type_names[0]의 type은 유니코드인데
        # s3에 올리기 위해서는 string이어야 하기 때문에 "utf-8"로 인코딩
        upload_file_name = saved_filename.encode("utf-8")

        # *S3 업로드 설정*
        # s3에 올리고자 하는 파일의 이름(폴더에 저장된 파일의 이름)
        filename = upload_file_name

        # aws s3에서 저장을 원하는 bucket 지정
        bucket_name = 'flexadstest'

        # s3에서의 objectname 설정
        objectname = filename
        # objectname = 'FlexAds_logo_init.png'

        # upload the file to AWS S3
        # 업로드 할 파일의 이름을 지정할 때, 본 파일과 같은 경로에 저장할 수도 있기 떄문에
        # 파일 이름에 경로까지 지정해주어야 업로드가 정상적으로 작동함
        s3.upload_file(watch_path + filename, bucket_name, objectname)


import pytest
import cv2
from plantcv.plantcv import rgb2gray_cmyk

def test_rgb2gray_cmyk(test_data):
    """Test for PlantCV."""
    # Read in test data
    img = cv2.imread(test_data.small_rgb_img)
    gray_img = rgb2gray_cmyk(rgb_img=img, channel="c")
    # Assert that the output image has the dimensions of the input image but is only a single channel
    assert img.shape[:2] == gray_img.shape

def test_rgb2gray_cmyk_bad_channel(test_data):
    """Test for PlantCV."""
    # Read in test data
    img = cv2.imread(test_data.small_rgb_img)
    with pytest.raises(RuntimeError):
        # Channel S is not in CMYK
        _ = rgb2gray_cmyk(rgb_img=img, channel="s")


from rest_framework import serializers

from api.user.models.Department import Department
from api.user.models.Teacher import Teacher
from api.user.models.User import User
from api.user.serializers.User import UserListSerializer

class UserMinSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ['identification_number', 'name', 'last_name']

class DepartmentSerializer(serializers.ModelSerializer):
    class Meta:
        model = Department
        fields = ['id', 'number', 'name']

class TeacherListSerializer(serializers.ModelSerializer):
    person = UserListSerializer()
    department = DepartmentSerializer()

    class Meta:
        model = Teacher
        fields = ['id','number', 'person', 'department']

class TeacherSerializer(serializers.ModelSerializer):

    class Meta:
        model = Teacher
        fields = ['number', 'person', 'department']


"""OAuth utility functions."""

import json
import re
from urllib.parse import quote_plus, urlparse

import structlog
from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter
from django.conf import settings
from oauthlib.oauth2.rfc6749.errors import InvalidGrantError, TokenExpiredError
from requests.exceptions import RequestException

from readthedocs.builds import utils as build_utils
from readthedocs.builds.constants import BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS
from readthedocs.integrations.models import Integration

from ..constants import GITLAB
from ..models import RemoteOrganization, RemoteRepository
from .base import Service, SyncServiceError

log = structlog.get_logger(__name__)

class GitLabService(Service):

    """
    Provider service for GitLab.

    See:
     - https://docs.gitlab.com/ce/integration/oauth_provider.html
     - https://docs.gitlab.com/ce/api/oauth2.html
    """

    adapter = GitLabOAuth2Adapter
    # Just use the network location to determine if it's a GitLab project
    # because private repos have another base url, eg. git@gitlab.example.com
    url_pattern = re.compile(
        re.escape(urlparse(adapter.provider_base_url).netloc),
    )

    PERMISSION_NO_ACCESS = 0
    PERMISSION_MAINTAINER = 40
    PERMISSION_OWNER = 50

    vcs_provider_slug = GITLAB

    def _get_repo_id(self, project):
        """
        Get the ID or URL-encoded path of the project.

        See https://docs.gitlab.com/ce/api/README.html#namespaced-path-encoding.
        """
        if project.remote_repository:
            repo_id = project.remote_repository.remote_id
        else:
            # Handle "Manual Import" when there is no RemoteRepository
            # associated with the project. It only works with gitlab.com at the
            # moment (doesn't support custom gitlab installations)
            username, repo = build_utils.get_gitlab_username_repo(project.repo)
            if (username, repo) == (None, None):
                return None

            repo_id = quote_plus(f"{username}/{repo}")
        return repo_id

    def get_next_url_to_paginate(self, response):
        return response.links.get("next", {}).get("url")

    def get_paginated_results(self, response):
        return response.json()

    def sync_repositories(self):
        remote_repositories = []
        try:
            repos = self.paginate(
                "{url}/api/v4/projects".format(url=self.adapter.provider_base_url),
                per_page=100,
                archived=False,
                order_by="path",
                sort="asc",
                membership=True,
            )

            for repo in repos:
                remote_repository = self.create_repository(repo)
                remote_repositories.append(remote_repository)
        except (TypeError, ValueError):
            log.warning("Error syncing GitLab repositories")
            raise SyncServiceError(
                SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format(
                    provider=self.vcs_provider_slug
                )
            )

        return remote_repositories

    def sync_organizations(self):
        remote_organizations = []
        remote_repositories = []

        try:
            orgs = self.paginate(
                "{url}/api/v4/groups".format(url=self.adapter.provider_base_url),
                per_page=100,
                all_available=False,
                order_by="path",
                sort="asc",
            )
            for org in orgs:
                remote_organization = self.create_organization(org)
                org_repos = self.paginate(
                    "{url}/api/v4/groups/{id}/projects".format(
                        url=self.adapter.provider_base_url,
                        id=org["id"],
                    ),
                    per_page=100,
                    archived=False,
                    order_by="path",
                    sort="asc",
                )

                remote_organizations.append(remote_organization)

                for repo in org_repos:
                    # TODO: Optimize this so that we don't re-fetch project data
                    # Details: https://github.com/readthedocs/readthedocs.org/issues/7743
                    try:
                        # The response from /groups/{id}/projects API does not contain
                        # admin permission fields for GitLab projects.
                        # So, fetch every single project data from the API
                        # which contains the admin permission fields.
                        resp = self.get_session().get(
                            "{url}/api/v4/projects/{id}".format(
                                url=self.adapter.provider_base_url, id=repo["id"]
                            )
                        )

                        if resp.status_code == 200:
                            repo_details = resp.json()
                            remote_repository = self.create_repository(
                                repo_details, organization=remote_organization
                            )
                            remote_repositories.append(remote_repository)
                        else:
                            log.warning(
                                "GitLab project does not exist or user does not have permissions.",
                                repository=repo["name_with_namespace"],
                            )

                    except Exception:
                        log.exception(
                            "Error creating GitLab repository",
                            repository=repo["name_with_namespace"],
                        )

        except (TypeError, ValueError):
            log.warning("Error syncing GitLab organizations")
            raise SyncServiceError(
                SyncServiceError.INVALID_OR_REVOKED_ACCESS_TOKEN.format(
                    provider=self.vcs_provider_slug
                )
            )

        return remote_organizations, remote_repositories

    def create_repository(self, fields, privacy=None, organization=None):
        """
        Update or create a repository from GitLab API response.

        ``admin`` field is computed using the ``permissions`` fields from the
        repository response. The permission from GitLab is given by an integer:
          * 0: No access
          * (... others ...)
          * 40: Maintainer
          * 50: Owner

        https://docs.gitlab.com/ee/api/access_requests.html
        https://gitlab.com/help/user/permissions

        :param fields: dictionary of response data from API
        :param privacy: privacy level to support
        :param organization: remote organization to associate with
        :type organization: RemoteOrganization
        :rtype: RemoteRepository
        """
        privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL
        repo_is_public = fields["visibility"] == "public"
        if privacy == "private" or (repo_is_public and privacy == "public"):
            repo, _ = RemoteRepository.objects.get_or_create(
                remote_id=fields["id"], vcs_provider=self.vcs_provider_slug
            )
            remote_repository_relation = repo.get_remote_repository_relation(
                self.user, self.account
            )

            if repo.organization and repo.organization != organization:
                log.debug(
                    "Not importing because mismatched orgs",
                    repository=fields["name"],
                )
                return None

            repo.organization = organization
            repo.name = fields["name"]
            repo.full_name = fields["path_with_namespace"]
            repo.description = fields["description"]
            repo.ssh_url = fields["ssh_url_to_repo"]
            repo.html_url = fields["web_url"]
            repo.vcs = "git"
            repo.private = not repo_is_public
            repo.default_branch = fields.get("default_branch")

            owner = fields.get("owner") or {}
            repo.avatar_url = fields.get("avatar_url") or owner.get("avatar_url")

            if not repo.avatar_url:
                repo.avatar_url = self.default_user_avatar_url

            if repo.private:
                repo.clone_url = repo.ssh_url
            else:
                repo.clone_url = fields["http_url_to_repo"]

            repo.save()

            project_access_level = group_access_level = self.PERMISSION_NO_ACCESS

            project_access = fields.get("permissions", {}).get("project_access", {})
            if project_access:
                project_access_level = project_access.get(
                    "access_level", self.PERMISSION_NO_ACCESS
                )

            group_access = fields.get("permissions", {}).get("group_access", {})
            if group_access:
                group_access_level = group_access.get(
                    "access_level", self.PERMISSION_NO_ACCESS
                )

            remote_repository_relation.admin = any(
                [
                    project_access_level
                    in (self.PERMISSION_MAINTAINER, self.PERMISSION_OWNER),
                    group_access_level
                    in (self.PERMISSION_MAINTAINER, self.PERMISSION_OWNER),
                ]
            )
            remote_repository_relation.save()

            return repo

        log.info(
            "Not importing repository because mismatched type.",
            repository=fields["path_with_namespace"],
            visibility=fields["visibility"],
        )

    def create_organization(self, fields):
        """
        Update or create remote organization from GitLab API response.

        :param fields: dictionary response of data from API
        :rtype: RemoteOrganization
        """
        organization, _ = RemoteOrganization.objects.get_or_create(
            remote_id=fields["id"], vcs_provider=self.vcs_provider_slug
        )
        organization.get_remote_organization_relation(self.user, self.account)

        organization.name = fields.get("name")
        organization.slug = fields.get("path")
        organization.url = "{url}/{path}".format(
            url=self.adapter.provider_base_url,
            path=fields.get("path"),
        )
        organization.avatar_url = fields.get("avatar_url")

        if not organization.avatar_url:
            organization.avatar_url = self.default_user_avatar_url

        organization.save()

        return organization

    def get_webhook_data(self, repo_id, project, integration):
        """
        Get webhook JSON data to post to the API.

        See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook
        """
        return json.dumps(
            {
                "id": repo_id,
                "push_events": True,
                "tag_push_events": True,
                "url": self.get_webhook_url(project, integration),
                "token": integration.secret,
                # Optional
                "issues_events": False,
                "merge_requests_events": True,
                "note_events": False,
                "job_events": False,
                "pipeline_events": False,
                "wiki_events": False,
            }
        )

    def get_provider_data(self, project, integration):
        """
        Gets provider data from GitLab Webhooks API.

        :param project: project
        :type project: Project
        :param integration: Integration for the project
        :type integration: Integration
        :returns: Dictionary containing provider data from the API or None
        :rtype: dict
        """

        if integration.provider_data:
            return integration.provider_data

        repo_id = self._get_repo_id(project)

        if repo_id is None:
            return None

        session = self.get_session()
        log.bind(
            project_slug=project.slug,
            integration_id=integration.pk,
        )

        rtd_webhook_url = self.get_webhook_url(project, integration)

        try:
            resp = session.get(
                "{url}/api/v4/projects/{repo_id}/hooks".format(
                    url=self.adapter.provider_base_url,
                    repo_id=repo_id,
                ),
            )

            if resp.status_code == 200:
                recv_data = resp.json()

                for webhook_data in recv_data:
                    if webhook_data["url"] == rtd_webhook_url:
                        integration.provider_data = webhook_data
                        integration.save()

                        log.info(
                            "GitLab integration updated with provider data for project.",
                        )
                        break
            else:
                log.info(
                    "GitLab project does not exist or user does not have permissions."
                )

        except Exception:
            log.exception("GitLab webhook Listing failed for project.")

        return integration.provider_data

    def setup_webhook(self, project, integration=None):
        """
        Set up GitLab project webhook for project.

        :param project: project to set up webhook for
        :type project: Project
        :param integration: Integration for a project
        :type integration: Integration
        :returns: boolean based on webhook set up success
        :rtype: bool
        """
        resp = None

        if not integration:
            integration, _ = Integration.objects.get_or_create(
                project=project,
                integration_type=Integration.GITLAB_WEBHOOK,
            )

        if not integration.secret:
            integration.recreate_secret()

        repo_id = self._get_repo_id(project)
        url = f"{self.adapter.provider_base_url}/api/v4/projects/{repo_id}/hooks"

        if repo_id is None:
            # Set the secret to None so that the integration can be used manually.
            integration.remove_secret()
            return (False, resp)

        log.bind(
            project_slug=project.slug,
            integration_id=integration.pk,
            url=url,
        )
        data = self.get_webhook_data(repo_id, project, integration)
        session = self.get_session()
        try:
            resp = session.post(
                url,
                data=data,
                headers={"content-type": "application/json"},
            )
            log.bind(http_status_code=resp.status_code)

            if resp.status_code == 201:
                integration.provider_data = resp.json()
                integration.save()
                log.debug("GitLab webhook creation successful for project.")
                return (True, resp)

            if resp.status_code in [401, 403, 404]:
                log.info(
                    "Gitlab project does not exist or user does not have permissions."
                )
            else:
                log.warning(
                    "GitLab webhook creation failed. Unknown response from GitLab."
                )

        except Exception:
            log.exception("GitLab webhook creation failed.")

        # Always remove secret and return False if we don't return True above
        integration.remove_secret()
        return (False, resp)

    def update_webhook(self, project, integration):
        """
        Update webhook integration.

        :param project: project to set up webhook for
        :type project: Project

        :param integration: Webhook integration to update
        :type integration: Integration

        :returns: boolean based on webhook update success, and requests Response
                  object

        :rtype: (Bool, Response)
        """
        provider_data = self.get_provider_data(project, integration)

        # Handle the case where we don't have a proper provider_data set
        # This happens with a user-managed webhook previously
        if not provider_data:
            return self.setup_webhook(project, integration)

        resp = None
        session = self.get_session()
        repo_id = self._get_repo_id(project)

        if repo_id is None:
            return (False, resp)

        if not integration.secret:
            integration.recreate_secret()

        data = self.get_webhook_data(repo_id, project, integration)

        log.bind(
            project_slug=project.slug,
            integration_id=integration.pk,
        )
        try:
            hook_id = provider_data.get("id")
            resp = session.put(
                "{url}/api/v4/projects/{repo_id}/hooks/{hook_id}".format(
                    url=self.adapter.provider_base_url,
                    repo_id=repo_id,
                    hook_id=hook_id,
                ),
                data=data,
                headers={"content-type": "application/json"},
            )

            if resp.status_code == 200:
                recv_data = resp.json()
                integration.provider_data = recv_data
                integration.save()
                log.info("GitLab webhook update successful for project.")
                return (True, resp)

            # GitLab returns 404 when the webhook doesn't exist. In this case,
            # we call ``setup_webhook`` to re-configure it from scratch
            if resp.status_code == 404:
                return self.setup_webhook(project, integration)

        except Exception:
            try:
                debug_data = resp.json()
            except ValueError:
                debug_data = resp.content
            except Exception:
                debug_data = None
            log.exception(
                "GitLab webhook update failed.",
                debug_data=debug_data,
            )

        integration.remove_secret()
        return (False, resp)

    def send_build_status(self, build, commit, status):
        """
        Create GitLab commit status for project.

        :param build: Build to set up commit status for
        :type build: Build
        :param status: build status failure, pending, or success.
        :type status: str
        :param commit: commit sha of the pull request
        :type commit: str
        :returns: boolean based on commit status creation was successful or not.
        :rtype: Bool
        """
        resp = None
        session = self.get_session()
        project = build.project

        repo_id = self._get_repo_id(project)

        if repo_id is None:
            return (False, resp)

        # select the correct status and description.
        gitlab_build_state = SELECT_BUILD_STATUS[status]["gitlab"]
        description = SELECT_BUILD_STATUS[status]["description"]

        if status == BUILD_STATUS_SUCCESS:
            # Link to the documentation for this version
            target_url = build.version.get_absolute_url()
        else:
            # Link to the build detail's page
            target_url = build.get_full_url()

        context = f"{settings.RTD_BUILD_STATUS_API_NAME}:{project.slug}"

        data = {
            "state": gitlab_build_state,
            "target_url": target_url,
            "description": description,
            "context": context,
        }
        url = f"{self.adapter.provider_base_url}/api/v4/projects/{repo_id}/statuses/{commit}"

        log.bind(
            project_slug=project.slug,
            commit_status=gitlab_build_state,
            user_username=self.user.username,
            url=url,
        )
        try:
            resp = session.post(
                url,
                data=json.dumps(data),
                headers={"content-type": "application/json"},
            )

            log.bind(http_status_code=resp.status_code)
            if resp.status_code == 201:
                log.debug("GitLab commit status created for project.")
                return True

            if resp.status_code in [401, 403, 404]:
                log.info(
                    "GitLab project does not exist or user does not have permissions."
                )
                return False

            return False

        # Catch exceptions with request or deserializing JSON
        except (RequestException, ValueError):
            # Response data should always be JSON, still try to log if not
            # though
            if resp is not None:
                try:
                    debug_data = resp.json()
                except ValueError:
                    debug_data = resp.content
            else:
                debug_data = resp

            log.exception(
                "GitLab commit status creation failed.",
                debug_data=debug_data,
            )
        except InvalidGrantError:
            log.info("Invalid GitLab grant for user.", exc_info=True)
        except TokenExpiredError:
            log.info("GitLab token expired for user.", exc_info=True)

        return False


import re
from urllib import robotparser
from urllib.parse import urljoin, urlparse
import requests

USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"

def download(url, num_retries=2, user_agent=USER_AGENT, proxies=None):
    print('Downloading:', url)
    headers = {'User-Agent': user_agent}
    try:
        resp = requests.get(url, headers=headers, proxies=proxies)
        html = resp.text
        if resp.status_code >= 400:
            print('Download error:', resp.text)
            html = None
            if num_retries and 500 <= resp.status_code < 600:
                # recursively retry 5xx HTTP errors
                return download(url, num_retries - 1)
    except requests.exceptions.RequestException as e:
        print('Download error:', e)
        html = None
    return html

def get_robots_parser(robots_url):
    " Return the robots parser object using the robots_url "
    rp = robotparser.RobotFileParser()
    rp.set_url(robots_url)
    rp.read()
    return rp

def get_links(html):
    webpage_regex = re.compile("""<a[^>]+href=["'](.*?)["']""", re.IGNORECASE)
    return webpage_regex.findall(html)

def jsonify(list1, list2):
    return {k:v for k,v in list(zip(list1, list2))}


from functools import total_ordering
import sys
sys.path.append('../')
import cv2
import os
import numpy as np
import mtcnn
from mtcnn.utils import draw
from numpy.linalg import inv, lstsq
from numpy.linalg import matrix_rank as rank

#初始化人脸对齐前后的路径
source_dir = '/home/fxf/my_model/resnet50/resnet50_test/test_images'
save_dir=source_dir+"_aligned"
os.mkdir(save_dir)

#112*112的五点标准位置，如果是112*96，则左边一列的每个坐标减8
REFERENCE_FACIAL_POINTS = np.array([
    [38.29459953, 51.69630051],
    [73.53179932, 51.50139999],
    [56.02519989, 71.73660278],
    [41.54930115, 92.3655014 ],
    [70.72990036, 92.20410156]
], np.float32)

#使用最小二乘法进行仿射变换
def findNonreflectiveSimilarity(uv, xy, K=2):

    M = xy.shape[0]
    x = xy[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
    y = xy[:, 1].reshape((-1, 1))  # use reshape to keep a column vector

    tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1))))
    tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1))))
    X = np.vstack((tmp1, tmp2))

    u = uv[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
    v = uv[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
    U = np.vstack((u, v))

    # We know that X * r = U
    if rank(X) >= 2 * K:
        r, _, _, _ = lstsq(X, U)
        r = np.squeeze(r)
    else:
        raise Exception('cp2tform:twoUniquePointsReq')

    sc = r[0]
    ss = r[1]
    tx = r[2]
    ty = r[3]

    Tinv = np.array([
        [sc, -ss, 0],
        [ss,  sc, 0],
        [tx,  ty, 1]
    ])
    T = inv(Tinv)
    T[:, 2] = np.array([0, 0, 1])
    T = T[:, 0:2].T

    return T

#获取权重
pnet, rnet, onet = mtcnn.get_net_caffe('../output/converted')

# 创建检测器
detector = mtcnn.FaceDetector(pnet, rnet, onet, device='cuda:0')  #使用gpu时还需指定设备号:"cuda:0"     使用cpu:"cpu"

#特征提取
dir_class = next(os.walk(source_dir))[1]
img_class = next(os.walk(source_dir))[2]
dir_class.sort()
img_class.sort()

#对目录下的子目录照片进行人脸对齐
print("正在进行对齐运算中……")
success_num=0
fail_num=0
for dir in dir_class:
    index=1
    img_dir=os.path.join(source_dir,dir)
    img_dir_aligned=os.path.join(save_dir,dir)
    os.mkdir(img_dir_aligned)
    for img in os.listdir(img_dir):
        format=img.split(".")[1]
        img_rename="/"+dir+"_"+str(index)+"."+format
        img = cv2.imread(os.path.join(img_dir,img))
        boxes, landmarks = detector.detect(img, minsize=24)
        if(len(landmarks)==0):
            print(img_rename,"对齐运算失败")
            fail_num+=1
            continue
        landmark=landmarks[0]
        similar_trans_matrix = findNonreflectiveSimilarity(landmark.cuda().numpy().astype(np.float32), REFERENCE_FACIAL_POINTS)
        aligned_img = cv2.warpAffine(img.copy(), similar_trans_matrix, (112, 112))
        cv2.imwrite(img_dir_aligned+img_rename, aligned_img)
        print("文件已成功保存到",img_dir_aligned+img_rename)
        index+=1
    success_num=success_num+index

#对目录下的照片文件进行人脸对齐
for img in img_class:
    img_rename="/"+img
    img = cv2.imread(os.path.join(source_dir,img))
    boxes, landmarks = detector.detect(img, minsize=24)
    landmark=landmarks[0]
    similar_trans_matrix = findNonreflectiveSimilarity(landmark.cpu().numpy().astype(np.float32), REFERENCE_FACIAL_POINTS)
    aligned_img = cv2.warpAffine(img.copy(), similar_trans_matrix, (112, 112))
    cv2.imwrite(save_dir+img_rename, aligned_img)
    print("文件已成功保存到",save_dir+img_rename)
    success_num=success_num+1

print("人脸对齐运算完毕！")
print("成功:",success_num)
print("失败:",fail_num)