import time
import torch
import utils_fns

from utils_general import show_with_error, plt_errors
from utils_mitsuba import get_mts_rendering, render_smooth

def run_optimization(hparams,
                     optim,
                     theta,
                     gt_theta,
                     ctx_args,
                     schedule_fn,
                     update_fn,
                     plot_initial,
                     plot_interval,
                     plot_intermediate,
                     print_param=False):
    sigma = hparams['sigma']

    reference_image = get_mts_rendering(gt_theta, update_fn, ctx_args)
    initial_image = get_mts_rendering(theta, update_fn, ctx_args)
    ctx_args['gt_image'] = reference_image

    # --------------- set up smoothed renderer
    smooth_mts = utils_fns.smoothFn(render_smooth,
                                    context_args=None,
                                    device=ctx_args['device'])

    if plot_initial:
        show_with_error(initial_image, reference_image, 0)

    img_errors, param_errors = [], []
    img_errors.append(torch.nn.MSELoss()(initial_image, reference_image).item())
    print(f"Running {hparams['epochs']} epochs with {hparams['nsamples']} samples and sigma={hparams['sigma']}")

    # --------------- run optimization
    for j in range(hparams['epochs']):
        start = time.time()
        optim.zero_grad()

        loss, _ = smooth_mts(theta.unsqueeze(0), ctx_args)
        loss.backward()

        optim.step()

        # potential sigma scheduling:
        if j > hparams['anneal_const_first'] and hparams['sigma_annealing'] and sigma >= hparams['anneal_sigma_min']:
            sigma = schedule_fn(sigma, curr_iter=j + 1, n=hparams['epochs'],
                                sigma_initial=hparams['sigma'],
                                sigma_min=hparams['anneal_sigma_min'],
                                const_first_n=hparams['anneal_const_first'],
                                const_last_n=hparams['anneal_const_last'])
            ctx_args['sigma'] = sigma
        iter_time = time.time() - start

        # logging, timing, plotting, etc...
        with torch.no_grad():

            # calc loss btwn rendering with current parameter (non-blurred)
            img_curr = get_mts_rendering(theta, update_fn, ctx_args)
            img_errors.append(torch.nn.MSELoss()(img_curr, ctx_args['gt_image']).item())
            param_errors.append(torch.nn.MSELoss()(theta, gt_theta).item())

            # plot intermediate
            if j % plot_interval == 0 and j > 0 and plot_intermediate:

                show_with_error(img_curr, ctx_args['gt_image'], j)

                if len(param_errors) > 1:
                    plt_errors(img_errors, param_errors, title=f'Ep {j + 1}')

            pstring = ' - CurrentParam: {}'.format(theta.tolist()) if print_param else ''
            print(f"Iter {j + 1}/{hparams['epochs']}, ParamLoss: {param_errors[-1]:.6f}, "
                  f"ImageLoss: {img_errors[-1]:.8f} - Time: {iter_time:.4f}{pstring}")

    plt_errors(img_errors, param_errors, title=f'Final, after {hparams["epochs"]} iterations')
    show_with_error(img_curr, ctx_args['gt_image'], hparams['epochs'])
    print("Done.")


from django.db import models

class Invoice(models.Model):
    invoice = models.OneToOneField(
        'mm.Invoice',
        verbose_name='发票',
    )
    invoice_code = models.CharField('发票号码', max_length=12, unique=True)
    date = models.DateField('开票日期', null=True)
    tracking_number = models.CharField('快递单号', max_length=15, blank=True)
    send_date = models.DateField('寄出日期', null=True)
    income_date = models.DateField('到账日期', null=True)

    class Meta:
        verbose_name = '发票管理'
        verbose_name_plural = '发票管理'

    def __str__(self):
        return '%s' % self.invoice_code

class Bill(models.Model):
    invoice = models.ForeignKey(Invoice, verbose_name='发票')
    income = models.DecimalField('到账金额', max_digits=9, decimal_places=2)
    date = models.DateField('到账日期')

    class Meta:
        verbose_name = '进账管理'
        verbose_name_plural = '进帐管理'

    def __str__(self):
        return '%f' % self.income


from models.markov_model import MarkovModel
from utils.globals import RC_FAILED, RC_SUCCESS
from utils.FASTA_parser import FastaParser

class IMM(object):
    """
    Class representation of the Interpolated Markov Model
    """
    def __init__(self, order, label, file):
        """
        Class constructor
        :param order: max order of MM members
        :param label: weight for each order
        :param file: path to FASTA file containing sequences, all with the same label/
        """
        self.orders = order  # List of Markov model orders
        self.weights = []  # List of corresponding weights
        self.models = [MarkovModel(i) for i in range(1, order+1, 1)]
        self.label = label
        self.file = file

    def train(self):
        training_set = FastaParser.get_sequences_from_file(self.file)
        for model in self.models:
            model.train(training_set)

    def predict(self, context):
        probabilities = []
        for model in self.models:
            probabilities.append(model.predict(context))
        interpolated_probability = sum(p * w for p, w in zip(probabilities, self.weights))
        return interpolated_probability

    def evaluate_success_rate(self, test_data):
        correct_predictions = 0
        total_predictions = 0
        for sequence in test_data:
            for i in range(self.orders[-1], len(sequence)):
                context = sequence[i - self.orders[-1]:i]
                actual_next_state = sequence[i]
                predicted_probability = self.predict(context)
                predicted_next_state = max(predicted_probability, key=predicted_probability.get)
                if predicted_next_state == actual_next_state:
                    correct_predictions += 1
                total_predictions += 1
        success_rate = correct_predictions / total_predictions
        return success_rate

    def evaluate_error_rate(self, test_data):
        return 1 - self.evaluate_success_rate(test_data)


import jwt
import os
from flask import request, g
from functools import wraps
from werkzeug.exceptions import Unauthorized

def auth_token():
    """Authenticate token and store user_id in g"""

    def _auth_token(func):
        @wraps(func)
        def __auth_token(*args, **kwargs):
            token = (
                request.headers.get("Authorization", "")
                .replace("Bearer ", "")
                .strip()
            )
            try:
                payload = jwt.decode(
                    token, os.environ["SECRET_KEY"], algorithms=["HS256"]
                )
            except jwt.exceptions.DecodeError:
                raise Unauthorized()
            g.user_id = payload.get("user_id")

            return func(*args, **kwargs)

        return __auth_token

    return _auth_token


import torch
import numpy as np

import torch_npu
from torch_npu.testing.testcase import TestCase, run_tests
import ads.common

class TestFastGelu(TestCase):
    def npu_op_exec(self, input1):
        input1.requires_grad = True
        output = ads.common.fast_gelu(input1)
        output.backward(torch.ones_like(output))
        output_grad = input1.grad
        output_grad = output_grad.to("cpu")
        output_grad = output_grad.detach().numpy()
        output = output.cpu().detach().numpy()
        return output_grad, output

    def test_fastgelu(self, device="npu"):
        input1 = torch.tensor([1., 2., 3., 4.]).npu()
        exoutputgrad = torch.tensor([1.0677795, 1.0738151, 1.0245483, 1.0064018])
        exoutput = torch.tensor([0.8458, 1.9357, 2.9819, 3.9956])
        outputgrad, output = self.npu_op_exec(input1)
        self.assertRtolEqual(exoutputgrad.numpy(), outputgrad)
        self.assertRtolEqual(exoutput.numpy(), output)

if __name__ == "__main__":
    run_tests()


"""
Generic template for paradigms linear classifiers on top of pretrained networks.
"""

from typing import Dict, Iterator, Optional, Union

import torch
import torch.nn as nn

from . import paradigm_utils

__all__ = [
    'epoch_loop',
    'test_dataset',
    'predict_dataset'
]

def epoch_loop(model: nn.Module, dataset: Iterator, optimiser: Union[torch.optim.Optimizer, None],
               device: Optional[torch.device] = None,
               return_outputs: Optional[bool] = False) -> Dict:
    """
    Executes an epoch of training or testing.

    Args:
        model: The neural network model to be trained or tested.
        dataset: The data loader providing data batches for training or testing.
        optimiser: The optimiser used for training (if None it switches to testing mode).
        device: The device to which the model and data will be transferred
         (default: CUDA if available) (optional).
        return_outputs: Whether to return model's outputs.

    Returns:
        A dictionary containing 'accuracy' and 'loss'.
        If `return_outputs` is True, it will also include the 'output' key, which contains the
        model's outputs for each data batch.
    """

    # Set the device
    if device is None:
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    # Determine whether the model is in training or evaluation mode
    is_train = False if optimiser is None else True

    # Ensure the model is in the appropriate mode (train or eval)
    model.train() if is_train else model.eval()
    model.to(device)

    # Initialise variables to store epoch-level statistics
    accuracies = []  # List of batch-wise accuracies
    losses = []  # List of batch-wise losses
    outputs = []  # List of batch-wise model outputs

    # Iterate over data batches
    with torch.set_grad_enabled(is_train):
        for batch_ind, batch_data in enumerate(dataset):
            # Transfer data to the device
            inputs = [img.to(device) for img in batch_data[:-1]]  # Input images
            target = batch_data[-1].to(device)  # Ground-truth labels

            # Forward pass the model
            output = model(*inputs)

            # Store outputs if requested
            if return_outputs:
                outputs.extend([out.detach().cpu().numpy() for out in output])

            # Compute loss
            loss = model.loss_function(output, target)
            # Append batch-wise losses
            losses.extend([loss.item() for _ in range(inputs[0].size(0))])
            # Compute accuracy
            accuracy = paradigm_utils.accuracy(output, target)
            # Append batch-wise accuracies
            accuracies.extend([accuracy for _ in range(inputs[0].size(0))])

            # Gradient computation and optimisation (if training mode)
            if is_train:
                optimiser.zero_grad()
                loss.backward()
                optimiser.step()

    # Create a dictionary containing epoch-level statistics
    epoch_log = {'accuracy': accuracies, 'loss': losses}

    # Add 'output' key if requested
    if return_outputs:
        epoch_log['output'] = outputs

    return epoch_log

def test_dataset(model: nn.Module, dataset: Iterator,
                 device: Optional[torch.device] = None) -> Dict:
    """
    Executes an epoch of testing.

    Args:
        model: The neural network model to be trained or tested.
        dataset: The data loader providing data batches for training or testing.
        device: The device to which the model and data will be transferred
         (default: CUDA if available) (optional).

    Returns:
        A dictionary containing 'accuracy' and 'loss'.
    """
    return epoch_loop(model, dataset, optimiser=None, device=device)

def predict_dataset(model: nn.Module, dataset: Iterator,
                    device: Optional[torch.device] = None) -> Dict:
    """
    Executes an epoch of prediction.

    Args:
        model: The neural network model to be trained or tested.
        dataset: The data loader providing data batches for training or testing.
        device: The device to which the model and data will be transferred
         (default: CUDA if available) (optional).

    Returns:
        A dictionary containing 'accuracy', 'loss' and 'output' (model's outputs).
    """
    return epoch_loop(model, dataset, optimiser=None, device=device, return_outputs=True)


from django.conf.urls import url
from . import views
urlpatterns = [
    url(r'^$', views.index),
    url(r'^loggedIn$', views.loggedIn),
    url(r'^logout$', views.logout),
    url(r'^register$', views.register),
    url(r'^registered$', views.registered),
    url(r'^dashboard$', views.dashboard),
    url(r'^home$', views.home),
    url(r'^reflection$', views.reflection),
    url(r'^essay$', views.essay),
    url(r'^delete/(?P<eid>\d+)$', views.delete),
    url(r'^location$', views.location)
]


from flask import Flask, render_template, request, redirect, url_for
from flask_sqlalchemy import SQLAlchemy
from flask_restx import Api, Resource, fields

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///exhibitions.db'
db = SQLAlchemy(app)
api = Api(app)

exhibition_model = api.model('Exhibition', {
    'id': fields.Integer(readonly=True, description='The unique identifier of an exhibition'),
    'name': fields.String(required=True, description='The name of the exhibition'),
    'date': fields.String(required=True, description='The date of the exhibition'),
    'location': fields.String(required=True, description='The location of the exhibition'),
    'participants': fields.Integer(required=True, description='The number of participants')
})

class Exhibition(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), nullable=False)
    date = db.Column(db.String(80), nullable=False)
    location = db.Column(db.String(120), nullable=False)
    participants = db.Column(db.Integer, nullable=False)

@api.route('/exhibitions')
class ExhibitionResource(Resource):
    @api.doc('list_exhibitions')
    @api.marshal_list_with(exhibition_model)
    def get(self):
        '''List all exhibitions'''
        exhibitions = Exhibition.query.all()
        return exhibitions

    @api.doc('create_exhibition')
    @api.expect(exhibition_model)
    @api.marshal_with(exhibition_model, code=201)
    def post(self):
        '''Create a new exhibition'''
        name = api.payload.get('name')
        date = api.payload.get('date')
        location = api.payload.get('location')
        participants = api.payload.get('participants')

        exhibition = Exhibition(name=name, date=date, location=location, participants=participants)
        db.session.add(exhibition)
        db.session.commit()

        return exhibition, 201

@api.route('/exhibitions/<int:id>')
@api.param('id', 'The exhibition identifier')
class ExhibitionItemResource(Resource):
    @api.doc('update_exhibition')
    @api.expect(exhibition_model)
    @api.marshal_with(exhibition_model)
    def put(self, id):
        '''Update an exhibition'''
        exhibition = Exhibition.query.get(id)
        if exhibition:
            exhibition.name = api.payload.get('name')
            exhibition.date = api.payload.get('date')
            exhibition.location = api.payload.get('location')
            exhibition.participants = api.payload.get('participants')
            db.session.commit()
            return exhibition
        else:
            return {'message': 'Exhibition not found'}, 404

    @api.doc('delete_exhibition')
    @api.response(204, 'Exhibition deleted')
    def delete(self, id):
        '''Delete an exhibition'''
        exhibition = Exhibition.query.get(id)
        if exhibition:
            db.session.delete(exhibition)
            db.session.commit()
            return '', 204
        else:
            return {'message': 'Exhibition not found'}, 404

if __name__ == '__main__':
    with app.app_context():
        db.create_all()
    app.run(debug=True)


#!/usr/bin/env python
import pika
import sys
import argparse
import pandas as pd
import json
import logging

logging.basicConfig(filename='publisher.log',level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")

parser = argparse.ArgumentParser(description='Data Ingest from CSV to CoreDms through stream')
parser.add_argument('-t','--tenant', help='Tenant ID', required=True)
parser.add_argument('-f','--file_path', help='Path of CSV file to export to CoreDMS', required=True)

args = vars(parser.parse_args())

if not all([args['tenant'], args['file_path']]):
    logging.error('Please select correct arrguments ')
    print('Please select correct arrguments')
    sys.exit()

tenant = args['tenant']
file_path = args['file_path']

connection = pika.BlockingConnection(
    pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='direct_ingestion', exchange_type='direct')

df = pd.read_csv(file_path)
records = json.loads(df.T.to_json()).values()
for record in records:
    logging.error("Message published: {}".format(json.dumps(record)))
    channel.basic_publish(
        exchange='direct_ingestion', routing_key=tenant, body=json.dumps(record))

connection.close()


import numpy as np
import pandas as pd
from scipy import stats
from sklearn.base import BaseEstimator, RegressorMixin, clone
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_predict
from sklearn.utils import check_X_y, check_array
from sklearn.utils.estimator_checks import check_estimator
from sklearn.utils.validation import check_is_fitted

class SuperLearner(BaseEstimator, RegressorMixin):
    def __init__(self, cand_learners=(LinearRegression(), RandomForestRegressor(n_estimators=10, random_state=1)),
                 meta_learner=LinearRegression(),
                 V=3):
        self.cand_learners = cand_learners
        self.meta_learner = meta_learner
        self.V = V

    def fit_cands(self, X, y):
        X, y = check_X_y(X, y)
        # step 1: split data into V blocks
        folds = [i % self.V for i in range(len(X))]
        # step 2-3: train each candidate learner and get CV predictions
        self.cand_learners_ = [clone(c) for c in self.cand_learners]
        Z = [cross_val_predict(cl, X, y, groups=folds, cv=self.V) for cl in self.cand_learners_]
        Z = np.transpose(Z)
        self.Z_train_cv_ = Z
        # step 0: fit candidate learners on whole dataset (have to do this after)
        for cand in self.cand_learners_:
            cand.fit(X, y)

        return Z

    def fit_meta(self, Z, y):
        Z = check_array(Z)
        self.meta_learner_ = clone(self.meta_learner)
        self.meta_learner_.fit(Z, y)

    def fit(self, X, y):
        # steps 0-3: fit candidate learners
        Z = self.fit_cands(X, y)
        # step 4: fit meta learner
        self.fit_meta(Z, y)

        return self

    def predict(self, X):
        check_is_fitted(self, "meta_learner_")
        X = check_array(X)
        Z = [cl.predict(X) for cl in self.cand_learners_]
        Z = np.transpose(Z)
        return self.meta_learner_.predict(Z)

    def mse(self, X, y):
        X, y = check_X_y(X, y)
        return mean_squared_error(y, self.predict(X))

    def get_cand_stats(self, X1, y1, X2=None, y2=None):
        stuff = []
        for cl, Z_cv in zip(self.cand_learners_, np.transpose(self.Z_train_cv_)):
            stuff.append([type(cl).__name__,
                          mean_squared_error(cl.predict(X1), y1),
                          mean_squared_error(Z_cv, y1)]
                         + ([mean_squared_error(cl.predict(X2), y2)] if X2 is not None else []))
        return stuff

    def get_meta_stats(self, X1, y1, X2=None, y2=None):
        return [f"Meta ({type(self.meta_learner_).__name__})",
                mean_squared_error(self.predict(X1), y1),
                mean_squared_error(self.meta_learner_.predict(self.Z_train_cv_), y1)] \
               + ([mean_squared_error(self.predict(X2), y2)] if X2 is not None else [])

    def debug(self, X1, y1, X2=None, y2=None, skip_fit=False, relative=False):
        """
        Fits on X1, y1 and predicts on X2, y2 and returns a DataFrame of useful info.
        X2, y2 optional.
        """
        X1, y1 = check_X_y(X1, y1)

        test = X2 is not None or y2 is not None
        if test:
            X2, y2 = check_X_y(X2, y2)

        if not skip_fit:
            self.fit(X1, y1)

        stuff = self.get_cand_stats(X1, y1, X2, y2)

        stuff.append(self.get_meta_stats(X1, y1, X2, y2))
        col_names = ["Learner", "Train MSE", "Train CV MSE"] + (["Test MSE"] if test else [])
        df = pd.DataFrame(data=stuff, columns=col_names)
        if relative:
            col_names.remove("Learner")
            col_mins = df[col_names].min(axis=0)
            df[col_names] = df[col_names]/col_mins
        try:
            df["Coefs"] = self.meta_learner_.coef_.tolist() + [sum(self.meta_learner_.coef_)]
        except:  # no coefs
            pass
        if relative:
            df = df.append(pd.Series(), ignore_index=True)
            df.iloc[-1,0] = 'Min Error'
            df.iloc[-1,1:len(col_mins)+1] = col_mins
        df = df.round(4)
        return df

def try_super_learners(cands, metas, X1, y1, X2, y2,relative=False):
    sl = SuperLearner(cand_learners=cands)
    sl.fit_cands(X1, y1)
    Z = sl.Z_train_cv_
    stats = sl.get_cand_stats(X1, y1, X2, y2)
    for meta in metas:
        sl.set_params(meta_learner=meta)
        sl.fit_meta(Z, y1)
        stats.append(sl.get_meta_stats(X1, y1, X2, y2))
    col_names = ["Learner", "Train MSE", "Train CV MSE"] + (["Test MSE"] if X2 is not None else [])
    df = pd.DataFrame(data=stats,
                      columns=col_names)
    if relative:
        col_names.remove("Learner")
        col_mins = df[col_names].min(axis=0)
        df[col_names] = df[col_names] / col_mins
        df = df.append(pd.Series(), ignore_index=True)
        df.iloc[-1, 0] = 'Min Error'
        df.iloc[-1, 1:len(col_mins) + 1] = col_mins
    df = df.round(4)
    return df

class BMA(BaseEstimator, RegressorMixin):
    def __init__(self, cand_learners=[LinearRegression()]):
        self.cand_learners = cand_learners

    def fit(self, X, y):
        X, y = check_X_y(X, y)
        n = len(X)
        p = len(np.transpose(X))
        self.cand_learners_ = [clone(c) for c in self.cand_learners]
        k = len(self.cand_learners_)
        BIC = np.zeros(k)
        norm_test = np.zeros((k,2))
        for cand, i in zip(self.cand_learners_, range(k)):
            cand.fit(X, y)
            # https://www.ssc.wisc.edu/~bhansen/718/NonParametrics15.pdf
            e_i = y - cand.predict(X)
            var_e_i = np.var(e_i) # this is hat sigma^2
            norm_test[i,] = stats.shapiro(e_i)
            ll_val = -0.5 * n * np.log(2 * np.pi * var_e_i) - n / 2 # simplified log likelihood
            BIC[i] = -2*ll_val + (p + 2) * np.log(n)

        # using log-sum-exp rule: https://www.xarg.org/2016/06/the-log-sum-exp-trick-in-machine-learning/
        logsum = max(-0.5*BIC) + np.log(np.sum(np.exp(-0.5*BIC - max(-0.5*BIC))))
        self.weights_ = np.exp(-0.5*BIC - logsum) # weighting formula
        self.norm_test_ = norm_test
        return self

    def weights(self):
        print([type(x).__name__ for x in self.cand_learners_])
        return self.weights_

    def norm_test(self, pval=True):
        if pval:
            return self.norm_test_[:,1]
        else:
            return self.norm_test_[:,0]

    def predict(self, X):
        check_is_fitted(self, 'weights_')
        X = check_array(X)
        y_hat = np.transpose([cl.predict(X) for cl in self.cand_learners_])
        return np.sum(y_hat * self.weights_, axis=1)

    def debug(self, X1, y1, X2=None, y2=None, skip_fit=False, relative=False):
        """
        Fits on X1, y1 and predicts on X2, y2 and returns a DataFrame of useful info.
        X2, y2 optional.
        """
        X1, y1 = check_X_y(X1, y1)

        test = X2 is not None or y2 is not None
        if test:
            X2, y2 = check_X_y(X2, y2)

        if not skip_fit:
            self.fit(X1, y1)

        stuff = []
        for cl in self.cand_learners_:
            stuff.append([type(cl).__name__,
                          mean_squared_error(cl.predict(X1), y1)]
                         + ([mean_squared_error(cl.predict(X2), y2)] if test else []))

        stuff.append(["BMA",
                      mean_squared_error(self.predict(X1), y1)]
                     + ([mean_squared_error(self.predict(X2), y2)] if test else []))

        col_names = ["Learner", "Train MSE"] + (["Test MSE"] if test else [])
        df = pd.DataFrame(data=stuff, columns=col_names)
        if relative:
            col_names.remove("Learner")
            col_mins = df[col_names].min(axis=0)
            df[col_names] = df[col_names]/col_mins
        try:
            df["Coefs"] = self.weights_.tolist() + [sum(self.weights_)]
            df["BIC"] = self.BIC_.tolist() + [None]
        except:  # no coefs
            pass
        if relative:
            df = df.append(pd.Series(), ignore_index=True)
            df.iloc[-1,0] = 'Min Error'
            df.iloc[-1,1:len(col_mins)+1] = col_mins
        df = df.round(4)
        return df

def gbm_summary(gbm, X1, y1, X2=None, y2=None):
    gbm.fit(X1,y1)
    test = X2 is not None or y2 is not None
    if test:
        X2, y2 = check_X_y(X2, y2)
    stuff = []
    stuff.append(["Gradient Boosting",
                  mean_squared_error(gbm.predict(X1), y1)]
                 + ([mean_squared_error(gbm.predict(X2), y2)] if test else []))
    col_names = ["Learner", "Train MSE"]  + (["Test MSE"] if test else [])
    params = pd.DataFrame.from_dict(gbm.best_params_,orient='index')
    print(params)
    df = pd.DataFrame(data=stuff, columns=col_names)
    df = df.round(4)
    return df

if __name__ == "__main__":
    check_estimator(SuperLearner())


import ubx
import gobject
import logging
import time

class Ublox(object):
    def __init__(self, device, quiet=True):
        self.loop = gobject.MainLoop()
        self.pollMaxRetries = 3
        self.quiet = quiet
        self.requestedMessageTypes = None
        self.message = None
        if device is not None:
            self.parser = ubx.Parser(self._waitForMessage, device=device)
        else:
            self.parser = ubx.Parser(self._waitForMessage)

    def setBaudRate(self, baudRate, baudRatesToTry=[9600, 115200]):
        for hostBaudRate in baudRatesToTry:
            # Set the host baud rate
            logging.info('Setting host baud rate to {}...'.format(hostBaudRate))
            self.parser.setBaudRate(hostBaudRate)
            # time.sleep(0.01)

            # Does this baud rate match the ublox?
            packet = self.poll('CFG-PRT')

            if packet is None:
                logging.info('No response at {} baud...'.format(hostBaudRate))
                continue

            logging.info('Received response at {} baud...'.format(hostBaudRate))
            if packet[1]['Baudrate'] == baudRate:
                logging.info('Baud rate is already set to the desired speed!')
                return True

            # Attempt to set the ublox baud rate
            newBaudRateRetries = 0
            while (newBaudRateRetries < 3):
                newBaudRateRetries += 1

                # Send CFG-PRT to configure the ublox baud rate
                logging.info('Attempting to configure the ublox to {} baud...'.format(baudRate))
                packet[1]['Baudrate'] = baudRate
                temp = self.sendConfig('CFG-PRT', 20, packet, maxRetries=1)
                time.sleep(0.1)

                # Set the host to the desired baud rate
                logging.info('Switching host baud rate to {}...'.format(baudRate))
                self.parser.setBaudRate(baudRate)

                # Poll for CFG-PRT
                newPacket = self.poll('CFG-PRT')
                if newPacket is not None:
                    if newPacket[1]['Baudrate'] == baudRate:
                        logging.info('Baud rate successfully set!')
                        return True
                else:
                    logging.info('Failed to set the ublox baud rate!')

                # Set the host to the desired baud rate
                logging.info('Resetting host baud rate to {}...'.format(hostBaudRate))
                self.parser.setBaudRate(hostBaudRate)

        raise Exception('Failed to set baud rate!')

    def poll(self, messageType, maxRetries=5, timeout=100):
        logging.info('Polling for {}...'.format(messageType))

        retries = 0
        while retries < maxRetries:
            self.message = None

            # Configure timeout
            timeoutSourceId = gobject.timeout_add(timeout, self._timeout)

            # Send poll message
            self.requestedMessageTypes = [messageType]
            self.parser.send(messageType, 0, [])

            # Look for response
            self.loop.run()

            # Clear timeout
            timeoutTriggered = self._clearTimeout(timeoutSourceId)

            if self.message is not None:
                ty, packet = self.message
                return packet

            retries += 1
            logging.info('retries: {}'.format(retries))
            time.sleep(0.1)

        return None

    def sendConfig(self, messageType, size, packet, maxRetries=5, timeout=100):
        logging.info('Sending {}...'.format(messageType))

        retries = 0
        while retries < maxRetries:
            self.message = None
            # Configure timeout
            timeoutSourceId = gobject.timeout_add(timeout, self._timeout)

            # Send config message
            self.requestedMessageTypes = ['ACK-ACK', 'ACK-NACK']
            self.parser.send(messageType, size, packet)

            # Look for response
            self.loop.run()

            # Clear timeout
            timeoutTriggered = self._clearTimeout(timeoutSourceId)

            if self.message is not None:
                ty, newPacket = self.message

                # Validate that the ACK is for our CFG message
                clsId, msgId = ubx.CLIDPAIR[ty]
                if newPacket[0]['ClsID'] != clsId or newPacket[0]['MsgID'] != msgId:
                    continue

                if ty == 'ACK-NACK':
                    raise Exception('ublox receiver responded with {}!'.format(ty))

                logging.info('Config message acknowledged by ublox.')
                return newPacket

            retries += 1
            # print('retries: {}'.format(retries))

        return None

    def _timeout(self):
        self.loop.quit()

    def _clearTimeout(self, sourceId):
        timeoutTriggered = True
        context = self.loop.get_context()
        if self.message is not None:
            gobject.source_remove(sourceId)
            timeoutTriggered = False
        return timeoutTriggered

    def _waitForMessage(self, ty, packet):
        logging.debug("Received %s" % repr([ty, packet]))
        if ty in self.requestedMessageTypes:
            self.message = ty, packet
            self.loop.quit()


import sys
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))

from time import time as now
from asyncratelimiter import AsyncRateLimiter
import asyncio
import pytest

async def foo(delay: float):
    await asyncio.sleep(delay)
    return 42

@pytest.mark.asyncio
async def test_async():
    limiter = AsyncRateLimiter(max_rate=5, interval=1)
    t0 = now()
    results = await asyncio.gather(*(limiter.dispatch(foo(0.1)) for _ in range(10)))
    assert now()-t0 > 1 and now()-t0 < 2
    assert all(map(lambda r: r == 42, results))

@pytest.mark.asyncio
async def test_context_managed():
    limiter = AsyncRateLimiter(max_rate=5, interval=1)
    t0 = now()
    for _ in range(10):
        async with limiter:
            await foo(0.1)
    assert now()-t0 > 1 and now()-t0 < 2


import argparse
import os
import copy

import numpy as np
import torch
from PIL import Image, ImageDraw, ImageFont

# Grounding DINO
import GroundingDINO.groundingdino.datasets.transforms as T
from GroundingDINO.groundingdino.models import build_model
from GroundingDINO.groundingdino.util import box_ops
from GroundingDINO.groundingdino.util.slconfig import SLConfig
from GroundingDINO.groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap

# segment anything
from segment_anything import build_sam, SamPredictor
import cv2
import numpy as np
import matplotlib.pyplot as plt

# diffusers
import PIL
import requests
import torch
from io import BytesIO
from stable_diffusion_masked_diffedit import StableDiffusionMaskedDiffeditPipeline
from diffusers import DDIMScheduler

# chatgpt
from chatgpt import call_chatgpt

# blip2
from transformers import AutoProcessor, Blip2ForConditionalGeneration

def load_image(image_path):
    # load image
    image_pil = Image.open(image_path).convert("RGB")  # load image

    transform = T.Compose(
        [
            T.RandomResize([800], max_size=1333),
            T.ToTensor(),
            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ]
    )
    image, _ = transform(image_pil, None)  # 3, h, w
    return image_pil, image

def load_model(model_config_path, model_checkpoint_path, device):
    args = SLConfig.fromfile(model_config_path)
    args.device = device
    model = build_model(args)
    checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
    load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
    print(load_res)
    _ = model.eval()
    return model

def get_grounding_output(model, image, caption, box_threshold, text_threshold, with_logits=True, device="cpu"):
    caption = caption.lower()
    caption = caption.strip()
    if not caption.endswith("."):
        caption = caption + "."
    model = model.to(device)
    image = image.to(device)
    with torch.no_grad():
        outputs = model(image[None], captions=[caption])
    logits = outputs["pred_logits"].cpu().sigmoid()[0]  # (nq, 256)
    boxes = outputs["pred_boxes"].cpu()[0]  # (nq, 4)
    logits.shape[0]

    # filter output
    logits_filt = logits.clone()
    boxes_filt = boxes.clone()
    filt_mask = logits_filt.max(dim=1)[0] > box_threshold
    logits_filt = logits_filt[filt_mask]  # num_filt, 256
    boxes_filt = boxes_filt[filt_mask]  # num_filt, 4
    logits_filt.shape[0]

    # get phrase
    tokenlizer = model.tokenizer
    tokenized = tokenlizer(caption)
    # build pred
    pred_phrases = []
    for logit, box in zip(logits_filt, boxes_filt):
        pred_phrase = get_phrases_from_posmap(logit > text_threshold, tokenized, tokenlizer)
        if with_logits:
            pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})")
        else:
            pred_phrases.append(pred_phrase)

    return boxes_filt, pred_phrases

def show_mask(mask, ax, random_color=False):
    if random_color:
        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
    else:
        color = np.array([30/255, 144/255, 255/255, 0.6])
    h, w = mask.shape[-2:]
    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
    ax.imshow(mask_image)

def show_box(box, ax, label):
    x0, y0 = box[0], box[1]
    w, h = box[2] - box[0], box[3] - box[1]
    ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2))
    ax.text(x0, y0, label)

def adjust_encoding_ratio(instructions, feedbacks, cur_encoding_ratio):
    full_instructions = []
    for instruction in instructions:
        full_instruction = '{}. The current encoding ratio is {}.'.format(instruction, cur_encoding_ratio)
        full_instructions.append(full_instruction)

    encoding_ratio = call_chatgpt(full_instructions, feedbacks=feedbacks)

    return encoding_ratio

if __name__ == "__main__":

    parser = argparse.ArgumentParser("Grounded-Segment-Anything Demo", add_help=True)
    parser.add_argument("--config", type=str, required=True, help="path to config file")
    parser.add_argument(
        "--grounded_checkpoint", type=str, required=True, help="path to checkpoint file"
    )
    parser.add_argument(
        "--sam_checkpoint", type=str, required=True, help="path to checkpoint file"
    )
    parser.add_argument("--input_image", type=str, required=True, help="path to image file")
    parser.add_argument("--user_instructions", type=str, required=True, help="user instructions")
    parser.add_argument("--is_blip2_description", action='store_false', default=True, help="to use blip2 to help the prompting or not")
    parser.add_argument("--is_show_mask", action='store_true', default=False, help="to save the dino mask as output or not")
    parser.add_argument(
        "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory"
    )
    parser.add_argument(
        "--output_name", type=str, default="output_image", help="output file name"
    )
    parser.add_argument("--cache_dir", type=str, default=None, help="save your huggingface large model cache")
    parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold")
    parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold")
    parser.add_argument("--encoding_ratio", type=float, default=0.5, help="encoding ratio for the image editing model")
    parser.add_argument("--inpaint_mode", type=str, default="first", help="inpaint mode")
    parser.add_argument("--device", type=str, default="cpu", help="running on cpu only!, default=False")
    args = parser.parse_args()

    # cfg
    config_file = args.config  # change the path of the model config file
    grounded_checkpoint = args.grounded_checkpoint  # change the path of the model
    sam_checkpoint = args.sam_checkpoint
    image_path = args.input_image
    user_instructions = args.user_instructions
    is_blip2_description = args.is_blip2_description
    is_show_mask = args.is_show_mask
    output_dir = args.output_dir
    output_name = args.output_name
    cache_dir=args.cache_dir
    box_threshold = args.box_threshold
    text_threshold = args.text_threshold
    encoding_ratio = args.encoding_ratio
    inpaint_mode = args.inpaint_mode
    device = args.device

    # load image
    image_pil, image = load_image(image_path)

    if is_blip2_description:
        blip_processor = AutoProcessor.from_pretrained('Salesforce/blip2-flan-t5-xl')
        blip_model = Blip2ForConditionalGeneration.from_pretrained('Salesforce/blip2-flan-t5-xl', torch_dtype=torch.float16).to(device)

        blip_prompt_1 = "Is this a photo, a painting, a drawing, or other kind of arts?"
        blip_inputs = blip_processor(image_pil, text=blip_prompt_1, return_tensors="pt").to(device, torch.float16)
        generated_ids = blip_model.generate(**blip_inputs, max_new_tokens=20)
        generated_text = blip_processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
        blip_prompt_2 = "{} of".format(generated_text.capitalize())
        blip_inputs = blip_processor(image_pil, text=blip_prompt_2, return_tensors="pt").to(device, torch.float16)
        generated_ids = blip_model.generate(**blip_inputs, max_new_tokens=20)
        description = blip_processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
        description = "{} {}.".format(blip_prompt_2, description)
        descriptions = [description]
        print('Blip2 description: {}'.format(description))
    else:
        descriptions = None
    instructions = [user_instructions]
    det_prompt, prompt_inversion, prompt, _ = call_chatgpt(instructions, descriptions=descriptions)
    print('Segmentation prompt: {}'.format(det_prompt))
    print('Inverted prompt: {}'.format(prompt_inversion))
    print('Editing prompt: {}'.format(prompt))

    # make dir
    os.makedirs(output_dir, exist_ok=True)

    # load model
    model = load_model(config_file, grounded_checkpoint, device=device)

    # run grounding dino model
    boxes_filt, pred_phrases = get_grounding_output(
        model, image, det_prompt, box_threshold, text_threshold, device=device
    )

    # initialize SAM
    predictor = SamPredictor(build_sam(checkpoint=sam_checkpoint).to(device))
    image = cv2.imread(image_path)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    predictor.set_image(image)

    size = image_pil.size
    H, W = size[1], size[0]
    for i in range(boxes_filt.size(0)):
        boxes_filt[i] = boxes_filt[i] * torch.Tensor([W, H, W, H])
        boxes_filt[i][:2] -= boxes_filt[i][2:] / 2
        boxes_filt[i][2:] += boxes_filt[i][:2]

    boxes_filt = boxes_filt.cpu()
    transformed_boxes = predictor.transform.apply_boxes_torch(boxes_filt, image.shape[:2]).to(device)

    masks, _, _ = predictor.predict_torch(
        point_coords = None,
        point_labels = None,
        boxes = transformed_boxes.to(device),
        multimask_output = False,
    )

    if is_show_mask:
        # draw output image
        plt.figure(figsize=(10, 10))
        plt.imshow(image)
        for mask in masks:
            show_mask(mask.cpu().numpy(), plt.gca(), random_color=True)

        plt.axis('off')
        plt.savefig(
            os.path.join(output_dir, "{}_mask.jpg".format(output_name)),
            bbox_inches="tight", dpi=300, pad_inches=0.0
        )

    # masks: [1, 1, 512, 512]

    # Masked diffedit pipeline
    if inpaint_mode == 'merge':
        masks = torch.sum(masks, dim=0).unsqueeze(0)
        masks = torch.where(masks > 0, True, False)
    mask = masks[0][0].cpu().numpy() # simply choose the first mask, which will be refine in the future release
    mask_pil = Image.fromarray(mask)
    image_pil = Image.fromarray(image)

    scheduler =DDIMScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", clip_sample=False,
                                        set_alpha_to_one=False)
    pipe = StableDiffusionMaskedDiffeditPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", scheduler=scheduler)
    pipe.to(device)

    image_pil = image_pil.resize((512, 512))
    mask_pil = mask_pil.resize((512, 512))
    image = pipe(prompt, image=image_pil, mask_image=mask_pil, prompt_inversion=prompt_inversion, encoding_ratio=encoding_ratio).images[0]
    image = image.resize(size)
    image.save(os.path.join(output_dir, "{}.jpg".format(output_name)))


from django.shortcuts import render, redirect
from django.core.urlresolvers import reverse
from django.http import JsonResponse
from django.http import HttpResponse
from django.views.generic import View
from photo.models import Photo
import time, datetime
from django.core.paginator import Paginator
# Create your views here.

class DayView(View):
    def get(self, request):
        all_photos_days = Photo.objects.all()
        photos_day = []
        for photo_day in all_photos_days:
            photo_create_time = str(photo_day.create_time).split(' ')[0]
            photo_day.photo_create_time = photo_create_time
            photos_day.append(photo_create_time)
            photos_day = list(set(photos_day))
            for i in all_photos_days:
                if photo_create_time not in photos_day:
                    photos_day.append(i)

            # photo_day = str(photo_day).split(' ')[0]
            # photos_day.append(photo_day)
        # photos_day = list(set(photos_day))
        # print(photos_day)
        # for first_photos in photos_day:
        #     first_photo = Photo.objects.filter(create_time__startswith=first_photos)[0]
        #     first_photos.first_photo = first_photo
        context = {'photos_day': photos_day}
        return render(request, 'day.html', context)

class PhotosView(View):
    def get(self, request, day, page):
        create_time = time.strftime('%Y-%m-%d', time.localtime())
        all_photos = Photo.objects.filter(create_time__startswith=day).order_by('-create_time')
        # all_photos = Photo.objects.filter(create_time__gt=datetime.date(2019,3,7)).order_by('-create_time')
        for photos in all_photos:
            image_id = (str(photos.image)).split('/')[-1]
            photos.image_id = image_id

        paginator = Paginator(all_photos, 5)
        try:
            page = int(page)
        except Exception as e:
            page = 1
        if page > paginator.num_pages:
            page = paginator.num_pages
        photos = paginator.page(page)
        num_pages = paginator.num_pages
        if num_pages < 5:
            pages = range(1, num_pages + 1)
        elif page <= 3:
            pages = range(1, 6)
        elif num_pages - page <= 2:
            pages = range(num_pages - 4, num_pages + 1)
        else:
            pages = range(page - 2, page + 3)

        context = {'all_photos': all_photos,
                   'day': day,
                   'photos': photos,
                   'pages': pages}
        return render(request, 'photos.html', context=context)

class  PhotoView(View):
    def get(self, request, image_id):
        image_id = 'group1/M00/00/00/'+image_id
        image = Photo.objects.get(image=image_id)
        content = {'image': image}
        print(content)
        return render(request, 'photos.html')


import os
from pathlib import Path
from typing import Any, Dict, List, Optional

import aws_cdk as cdk
import aws_cdk.aws_stepfunctions as sfn
from aws_cdk.aws_glue import CfnDatabase
from aws_cdk.aws_iam import Effect, PolicyStatement
from aws_cdk.aws_kms import IKey, Key
from aws_cdk.aws_lakeformation import CfnPermissions
from aws_cdk.aws_lambda import Code, Function, LayerVersion, Runtime
from aws_cdk.aws_s3 import Bucket, IBucket
from aws_cdk.aws_ssm import StringParameter
from aws_cdk.aws_stepfunctions import JsonPath
from aws_cdk.aws_stepfunctions_tasks import LambdaInvoke
from aws_ddk_core import StateMachineStage
from constructs import Construct

class DataValidationCatalogingStage(StateMachineStage):
    def __init__(
        self,
        scope: Construct,
        id: str,
        environment_id: str,
        raw_bucket_arn: str,
        didc_table_arn: str,
        didc_table_key_arn: str,
        state_machine_failed_executions_alarm_threshold: Optional[int] = 1,
        state_machine_failed_executions_alarm_evaluation_periods: Optional[int] = 1,
    ) -> None:
        super().__init__(scope, id)

        self._environment_id = environment_id
        self._raw_bucket_arn = raw_bucket_arn
        self._didc_table_arn = didc_table_arn
        self._didc_table_key_arn = didc_table_key_arn
        self._pipeline_id = "validation-cataloging"

        self._raw_bucket: IBucket = Bucket.from_bucket_arn(
            self, f"di-raw-bucket", bucket_arn=self._raw_bucket_arn
        )

        self._wrangler_layer = LayerVersion.from_layer_version_arn(
            self,
            id="layer",
            layer_version_arn=f"arn:aws:lambda:{cdk.Aws.REGION}:336392948345:layer:AWSDataWrangler-Python39:1",
        )

        self._didc_table_key: IKey = Key.from_key_arn(
            self, f"di-didc-table-key", key_arn=self._didc_table_key_arn
        )

        db_name = f"di-validation-cataloging-data-catalog-{self._environment_id}"
        self._database = self._create_database(database_name=db_name)

        state_machine_role_policy_statements = [
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=[
                    "lakeformation:GetDataAccess",
                    "lakeformation:GrantPermissions",
                    "glue:*Database*",
                    "glue:*Table*",
                    "glue:*Partition*",
                    "glue:SearchTables",
                    "states:StartExecution",
                    "glue:Get*",
                    "glue:Update*",
                    "glue:Start*",
                ],
                resources=["*"],
            ),
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=[
                    "kms:Decrypt",
                    "kms:Encrypt",
                    "kms:ReEncrypt*",
                    "kms:GenerateDataKey*",
                ],
                resources=["*"],
            ),
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=[
                    "s3:DeleteObject*",
                    "s3:Put*",
                    "s3:Abort*",
                    "s3:GetObject*",
                    "s3:GetBucket*",
                    "s3:List*",
                ],
                resources=["*"],
            ),
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["kms:Encrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*"],
                resources=["*"],
            ),
        ]

        # Schema Validation
        schema_validation = self._create_schema_validation(prefix="di")

        # Schema Registration
        schema_registration = self._create_schema_registration(prefix="di")

        # Object Tagging
        object_tagging = self._create_object_tagging(prefix="di")

        # Catalog Tagging
        catalog_tagging = self._create_catalog_tagging(prefix="di")

        # COPY S3 DATA
        s3_map = sfn.Map(
            self,
            f"di-{self._pipeline_id}-Copy",
            input_path="$.body",
            items_path="$.key_copy_source",
            max_concurrency=0,
            parameters={
                "target_bucket.$": "$.target_bucket",
                "target_prefix.$": "$.target_prefix",
                "Bucket.$": "$.bucket",
                "s3_object.$": "$$.Map.Item.Value",
            },
            result_path=JsonPath.DISCARD,
        )

        copy_s3_state_json = {
            "Type": "Task",
            "Resource": "arn:aws:states:::aws-sdk:s3:copyObject",
            "Parameters": {
                "Bucket.$": "$.target_bucket",
                "CopySource.$": "States.Format('{}/{}', $.Bucket, $.s3_object.full_path)",
                "Key.$": "States.Format('{}/{}', $.target_prefix, $.s3_object.filename)",
            },
            "End": True,
        }

        copy_s3_iterator = sfn.CustomState(
            self,
            f"di-{self._pipeline_id}-DataS3Copy-Iterator",
            state_json=copy_s3_state_json,
        )

        s3_map.iterator(copy_s3_iterator)

        # SUCCEED STATE
        success_state = sfn.Succeed(self, "Done")

        # FAIL STATE
        fail_state = sfn.Fail(self, "Fail", error="States.ALL")

        # CREATE PARALLEL STATE DEFINITION
        parallel_state = sfn.Parallel(self, "ParallelSM")

        parallel_state.branch(
            schema_validation.next(schema_registration)
            .next(object_tagging)
            .next(catalog_tagging)
            .next(s3_map)
        )

        parallel_state.next(success_state)

        parallel_state.add_catch(
            fail_state, errors=["States.ALL"], result_path="$.error"
        )

        # CREATE STATE MACHINE
        state_object = self._create_state_machine(
            definition=(parallel_state),
            additional_role_policy_statements=state_machine_role_policy_statements,
            state_machine_failed_executions_alarm_threshold=state_machine_failed_executions_alarm_threshold,
            state_machine_failed_executions_alarm_evaluation_periods=state_machine_failed_executions_alarm_evaluation_periods,  # noqa
        )

        self._event_pattern, self._targets, self._state_machine = (
            state_object.event_pattern,
            state_object.targets,
            state_object.state_machine,
        )

        self._create_table_lakeformation_perms(
            prefix="di",
            name="catalog",
            lambda_role_arn=self.catalog_tagging_role_arn,
            database_name=self._database.ref,
            env=self._environment_id,
        )

        self._create_db_lakeformation_perms(
            prefix="di",
            name="schema-registration",
            lambda_role_arn=self.schema_registration_role_arn,
            database_name=self._database.ref,
            env=self._environment_id,
        )

    @property
    def targets(self):
        return self._targets

    @property
    def event_pattern(self):
        return self._event_pattern

    @property
    def state_machine(self):
        return self._state_machine

    @property
    def state_machine_arn(self):
        return self._state_machine.state_machine_arn

    @property
    def catalog_tagging_role_arn(self):
        return self._catalog_tagging_lambda.role.role_arn

    @property
    def schema_registration_role_arn(self):
        return self._schema_registration.role.role_arn

    def _create_database(self, database_name: str):
        return CfnDatabase(
            self,
            database_name,
            database_input=CfnDatabase.DatabaseInputProperty(
                name=database_name,
                location_uri=f"s3://{self._raw_bucket.bucket_name}/",
            ),
            catalog_id=cdk.Aws.ACCOUNT_ID,
        )

    def _create_schema_registration(self, prefix: str) -> LambdaInvoke:
        self._schema_registration: Function = Function(
            self,
            id=f"{prefix}-{self._pipeline_id}-SchemaRegistration",
            function_name=f"{prefix}-{self._pipeline_id}-SchemaRegistration",
            code=Code.from_asset(
                os.path.join(f"{Path(__file__).parents[1]}", "src/schema_registration")
            ),
            handler="handler.lambda_handler",
            runtime=Runtime.PYTHON_3_9,
            environment={"ENV": self._environment_id, "PREFIX": "di"},
            description="validates incoming file schema with expected enterprise schema",
            timeout=cdk.Duration.minutes(15),
            layers=[self._wrangler_layer],
        )
        self._schema_registration.add_to_role_policy(
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["dynamodb:*"],
                resources=[
                    f"arn:aws:dynamodb:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:table/*"
                ],
            )
        )

        self._schema_registration.add_to_role_policy(
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["glue:*"],
                resources=["*"],
            )
        )
        self._didc_table_key.grant_decrypt(self._schema_registration)
        self._raw_bucket.grant_read(self._schema_registration)

        schema_registration = LambdaInvoke(
            self,
            f"{prefix}-{self._pipeline_id}-schemaRegistration-step",
            lambda_function=self._schema_registration,
            comment="Schema Registration",
            result_path=JsonPath.DISCARD,
        )

        schema_registration.add_retry(errors=["States.ALL"], max_attempts=2)

        return schema_registration

    def _create_schema_validation(self, prefix: str) -> LambdaInvoke:
        self._schema_validation: Function = Function(
            self,
            id=f"{prefix}-{self._pipeline_id}-SchemaValidation",
            function_name=f"{prefix}-{self._pipeline_id}-SchemaValidation",
            code=Code.from_asset(
                os.path.join(f"{Path(__file__).parents[1]}", "src/schema_validation")
            ),
            handler="handler.lambda_handler",
            runtime=Runtime.PYTHON_3_9,
            environment={"DIDC": self._didc_table_arn},
            description="validates incoming file schema with expected enterprise schema",
            timeout=cdk.Duration.minutes(15),
            layers=[self._wrangler_layer],
        )
        self._schema_validation.add_to_role_policy(
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["dynamodb:*"],
                resources=[
                    f"arn:aws:dynamodb:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:table/*"
                ],
            )
        )
        self._didc_table_key.grant_decrypt(self._schema_validation)
        self._raw_bucket.grant_read(self._schema_validation)

        schema_validation = LambdaInvoke(
            self,
            f"{prefix}-{self._pipeline_id}-schemaValidation-step",
            lambda_function=self._schema_validation,
            comment="Schema Validation",
            result_path=JsonPath.DISCARD,
        )
        schema_validation.add_retry(
            errors=["SCHEMA_MISMATCH_ERROR", "COUNT_NOT_MATCHED_ERROR"], max_attempts=0
        )
        schema_validation.add_retry(errors=["States.ALL"], max_attempts=2)

        return schema_validation

    def _create_object_tagging(self, prefix: str) -> LambdaInvoke:
        self._object_tagging_lambda: Function = Function(
            self,
            id=f"{prefix}-{self._pipeline_id}-ObjectTag",
            function_name=f"{prefix}-{self._pipeline_id}-ObjectTag",
            code=Code.from_asset(
                os.path.join(f"{Path(__file__).parents[1]}", "src/object_tagging")
            ),
            handler="handler.lambda_handler",
            runtime=Runtime.PYTHON_3_9,
            environment={
                "DIDC": self._didc_table_arn,
            },
            description="Tags object with metadata in s3",
            timeout=cdk.Duration.minutes(10),
        )

        self._object_tagging_lambda.add_to_role_policy(
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["dynamodb:*"],
                resources=[
                    f"arn:aws:dynamodb:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:table/*"
                ],
            )
        )
        self._didc_table_key.grant_decrypt(self._object_tagging_lambda)
        self._raw_bucket.grant_read_write(self._object_tagging_lambda)

        object_tagging = LambdaInvoke(
            self,
            f"{prefix}-{self._pipeline_id}-objectTagging-step",
            lambda_function=self._object_tagging_lambda,
            comment="Data Object Tagging",
            result_path=JsonPath.DISCARD,
        )

        object_tagging.add_retry(errors=["States.ALL"], max_attempts=2)

        return object_tagging

    def _create_catalog_tagging(self, prefix: str) -> LambdaInvoke:
        self._catalog_tagging_lambda: Function = Function(
            self,
            id=f"{prefix}-{self._pipeline_id}-CatalogTag",
            function_name=f"{prefix}-{self._pipeline_id}-CatalogTag",
            code=Code.from_asset(
                os.path.join(f"{Path(__file__).parents[1]}", "src/catalog_tagging")
            ),
            handler="handler.lambda_handler",
            runtime=Runtime.PYTHON_3_9,
            environment={
                "ENV": self._environment_id,
                "DIDC": self._didc_table_arn,
                "PREFIX": prefix,
            },
            description="Tags objects in glue catalog",
            timeout=cdk.Duration.minutes(10),
        )
        self._catalog_tagging_lambda.add_to_role_policy(
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["glue:*Database*", "glue:*Table*", "glue:*Partition*"],
                resources=["*"],
            )
        )
        self._catalog_tagging_lambda.add_to_role_policy(
            PolicyStatement(
                effect=Effect.ALLOW,
                actions=["dynamodb:*"],
                resources=[
                    f"arn:aws:dynamodb:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:table/*"
                ],
            )
        )
        self._didc_table_key.grant_decrypt(self._catalog_tagging_lambda)

        StringParameter(
            self,
            f"{prefix}-{self._pipeline_id}-catalog-lambda-role-{self._environment_id}",
            parameter_name=f"/DIF/{self._pipeline_id}/Lambda/CatalogLambdaRoleArn",
            string_value=self._catalog_tagging_lambda.role.role_arn,
        )

        catalog_tagging = LambdaInvoke(
            self,
            f"{prefix}-{self._pipeline_id}-catalogTagging-step",
            lambda_function=self._catalog_tagging_lambda,
            comment="Data Catalog Tagging",
            result_path=JsonPath.DISCARD,
        )

        catalog_tagging.add_retry(errors=["States.ALL"], max_attempts=2)

        return catalog_tagging

    def _create_table_lakeformation_perms(
        self, prefix, name, lambda_role_arn, database_name, env
    ):
        return CfnPermissions(
            self,
            f"{prefix}-{name}-lambda-database-lakeformation-permissions-{env}",
            data_lake_principal=CfnPermissions.DataLakePrincipalProperty(
                data_lake_principal_identifier=lambda_role_arn
            ),
            resource=CfnPermissions.ResourceProperty(
                table_resource=CfnPermissions.TableResourceProperty(
                    database_name=database_name, table_wildcard={}
                ),
            ),
            permissions=["SELECT", "ALTER", "INSERT", "DESCRIBE"],
        )

    def _create_db_lakeformation_perms(
        self, prefix, name, lambda_role_arn, database_name, env
    ):
        return CfnPermissions(
            self,
            f"{prefix}-{name}-database-lakeformation-permissions-{env}",
            data_lake_principal=CfnPermissions.DataLakePrincipalProperty(
                data_lake_principal_identifier=lambda_role_arn
            ),
            resource=CfnPermissions.ResourceProperty(
                database_resource=CfnPermissions.DatabaseResourceProperty(
                    name=database_name
                )
            ),
            permissions=["CREATE_TABLE", "ALTER", "DROP"],
        )


import random as rand
import time
from typing import List
import bs4
import requests as req
from data import Product, Website
import urllib3

# disable the insecure certificates warning on the console
urllib3.disable_warnings()

class UrlDownloader:
    __WINDOWS_BROWSER_REQUEST_PROPERTY = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11'
    __LINUX_BROWSER_REQUEST_PROPERTY = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'
    __MAC_BROWSER_REQUEST_PROPERTY = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
    __AMAZON_WEBSITE_NAME: str = 'Amazon'
    __ALDI_UK_WEBSITE_NAME: str = 'Aldi-UK'
    __TESCO_WEBSITE_NAME: str = 'Tesco'

    # free proxy list website url --> https://spys.one/ only get http proxies
    __proxies = [
        'http://84.17.35.129:3128',
        'http://45.32.109.170:8080',
        'http://84.54.217.19:8080'
    ]
    __session_proxies = {
        'http': rand.choice(__proxies)
    }
    __website_name_session_param = {
        __AMAZON_WEBSITE_NAME: req.Session(),
        __ALDI_UK_WEBSITE_NAME: req.Session(),
        __TESCO_WEBSITE_NAME: req.Session()
    }

    __validBrowserAgents = [__WINDOWS_BROWSER_REQUEST_PROPERTY, __LINUX_BROWSER_REQUEST_PROPERTY,
                            __MAC_BROWSER_REQUEST_PROPERTY]

    def scrap_items(self, website: Website, products: List) -> Website:
        website_name_product_html_lambda_switcher = {
            self.__ALDI_UK_WEBSITE_NAME: self.__retreive_aldi_uk_products_from_html,
            self.__AMAZON_WEBSITE_NAME: self.__retreive_amazon_products_from_html,
            self.__TESCO_WEBSITE_NAME: self.__retreive_tesco_products_from_html
        }

        aggregated_products = []

        for product in products:
            try:
                product_html = self.__fetch_product_html(product, website.search_query_url, website.name,
                                                         website.search_query_param)
                scrap_products_lambda = website_name_product_html_lambda_switcher.get(website.name, None)
                if scrap_products_lambda is None:
                    website.products = []
                    return website
                scrap_products = scrap_products_lambda(product_html, website.home_url)
                aggregated_products.extend(scrap_products)
            except Exception as exp:
                print(
                    f'Error occurred for website --> {website.name}, for product --> {product}, and the exception --> {exp}')
                continue

        temp_website = Website(website.name, website.home_url, website.search_query_url, website.search_query_param,
                               aggregated_products)
        self.__website_name_session_param[temp_website.name].close()
        return temp_website

    def __fetch_product_html(self, product_name: str, website_search_base_url: str, website_name: str,
                             website_query_param: str) -> str:
        header_agent = {'User-Agent': rand.choice(self.__validBrowserAgents), 'Accept-Language': 'en-US, en;q=0.5'}
        session = self.__website_name_session_param[website_name]
        query_param = {website_query_param: product_name}
        if len(session.proxies) == 0:
            session.proxies.update(self.__session_proxies)
        product_response = session.get(website_search_base_url, params=query_param, headers=header_agent,
                                       timeout=20, verify=False)
        random_request_delay = rand.randint(2, 3)
        time.sleep(random_request_delay)
        return product_response.text if product_response is not None else ''

    @staticmethod
    def __retreive_tesco_products_from_html(product_html: str, website_home_url: str) -> List:
        soup = bs4.BeautifulSoup(product_html, 'lxml')
        all_product_divs = soup.find_all('li', attrs={'class': 'product-list--list-item'})
        products = list()

        for single_product_div in all_product_divs:

            if single_product_div is None:
                continue

            # extracting the product title and and image base-64 string
            product_image_container_div: bs4.Tag = single_product_div.find('div',
                                                                           attrs={'class': 'product-image__container'})
            product_image_tag: bs4.Tag = product_image_container_div.find(
                'img') if product_image_container_div is not None else None
            product_title: str = product_image_tag.get('alt') if product_image_tag is not None else None
            if product_title is None or product_title == '':
                continue
            product_image_single_source_base64: str = product_image_tag.get(
                'src') if product_image_tag is not None else None

            # extracting the product detail link
            product_detail_content_div: bs4.Tag = single_product_div.find('div',
                                                                          attrs={'class': 'product-details--content'})
            product_detail_content_tag: bs4.Tag = product_detail_content_div.find(
                attrs={'data-auto': 'product-tile--title'}) if product_detail_content_div is not None else None
            product_detail_link: str = product_detail_content_tag.get(
                'href') if product_detail_content_tag is not None else None

            # extracting the product price details
            product_detail_wrapper_div: bs4.Tag = single_product_div.find('div',
                                                                          attrs={'class': 'price-details--wrapper'})
            product_currency_tag_span: bs4.Tag = product_detail_wrapper_div.find(
                attrs={'class': 'currency'}) if product_detail_wrapper_div is not None else None
            product_currency: str = product_currency_tag_span.text if product_currency_tag_span is not None else None
            product_price_tag_span: bs4.Tag = product_detail_wrapper_div.find(
                attrs={'data-auto': 'price-value'}) if product_detail_wrapper_div is not None else None
            product_price: str = product_price_tag_span.text if product_price_tag_span is not None else None
            whole_product_price = f'{product_currency}{product_price}' if product_currency is not None and product_price is not None else None
            price_unit = UrlDownloader.__retreive_product_unit_from_price(
                whole_product_price) if whole_product_price is not None else None
            product_price = UrlDownloader.__remove_product_unit_from_price(
                whole_product_price) if whole_product_price is not None else None

            # extracting the ounce pound text
            price_per_quantity_weight_div: bs4.Tag = single_product_div.find('div', attrs={
                'class': 'price-per-quantity-weight'})
            quantity_price_symbol_span: bs4.Tag = price_per_quantity_weight_div.find(
                attrs={'class': 'currency'}) if price_per_quantity_weight_div is not None else None
            quantity_price_symbol: str = quantity_price_symbol_span.text if quantity_price_symbol_span is not None else None
            quantity_price_span: bs4.Tag = price_per_quantity_weight_div.find(
                attrs={'data-auto': 'price-value'}) if price_per_quantity_weight_div is not None else None
            quantity_price: str = quantity_price_span.text if quantity_price_span is not None else None
            quantity_per_price_span: bs4.Tag = price_per_quantity_weight_div.find(
                attrs={'class': 'weight'}) if price_per_quantity_weight_div is not None else None
            quantity_per_price: str = quantity_per_price_span.text if quantity_per_price_span is not None else None
            ounce_pound_text = f'{quantity_price_symbol}{quantity_price}{quantity_per_price}' \
                if quantity_price_symbol is not None and quantity_price is not None and quantity_per_price is not None else None

            if product_image_single_source_base64 is None or product_detail_link is None or product_price is None or product_price == '':
                continue

            product = Product(product_title, product_image_single_source_base64, '',
                              product_detail_link if product_detail_link.startswith(
                                  'http') else website_home_url + product_detail_link, list(), ounce_pound_text,
                              product_price, '', price_unit)
            products.append(product)
        return products

    @staticmethod
    def __retreive_aldi_uk_products_from_html(product_html: str, website_home_url: str) -> List:
        soup = bs4.BeautifulSoup(product_html, 'lxml')
        all_product_divs = soup.select('div.category-item,div.js-category-item')
        products = list()
        for single_product_div in all_product_divs:

            if single_product_div is None:
                continue

            # extracting the product detail url
            a_link_class: bs4.Tag = single_product_div.find('a',
                                                            attrs={'class': 'category-item__link js-category-link'})
            product_detail_link: str = a_link_class.get('href') if a_link_class is not None else None
            if product_detail_link is None or product_detail_link == '':
                continue

            # extracting the product single image source and multiple image sources
            outer_picture_class: bs4.Tag = single_product_div.find('picture', attrs={
                'class': 'category-item__image js-category-image'})
            all_source_media_tags: bs4.ResultSet = outer_picture_class.find_all(
                'source') if outer_picture_class is not None else None
            product_image_source_set: List = []
            if all_source_media_tags is not None:
                for single_source_media_tag in all_source_media_tags:
                    comma_separated_source_set_attribute: str = single_source_media_tag.get('srcset')
                    try:
                        single_media_tag: str = comma_separated_source_set_attribute.split(',')[
                            1] if comma_separated_source_set_attribute is not None else None

                        # remove the 2x,3x,2.5x from featured images if we found the featured image

                        space_removal_single_media_tag: str = single_media_tag.strip() if single_media_tag is not None else None
                        if space_removal_single_media_tag is not None and ' ' in space_removal_single_media_tag:
                            product_image_source_set.append(space_removal_single_media_tag.split(' ')[0].strip())
                        elif space_removal_single_media_tag is not None:
                            product_image_source_set.append(space_removal_single_media_tag)
                    except ValueError as e:
                        print(e)
            product_image_single_source_tag = outer_picture_class.find(
                'img') if outer_picture_class is not None else None
            product_image_single_source_attribute_value: str = product_image_single_source_tag.get(
                'srcset') if product_image_single_source_tag is not None else None
            try:
                product_image_single_source = product_image_single_source_attribute_value.split(',')[
                    1] if product_image_single_source_attribute_value is not None else ''
            except ValueError:
                product_image_single_source = product_image_single_source_tag.get(
                    'src') if product_image_single_source_tag is not None else ''

            # extracting the product title
            product_title_class: bs4.Tag = single_product_div.find(attrs={'class': 'category-item__title'})
            product_title = product_title_class.find('a').text if product_title_class is not None else None

            # extracting the product price
            product_price_class: bs4.Tag = single_product_div.find(attrs={'class': 'category-item__price'})
            whole_product_price: str = product_price_class.text if product_price_class is not None else None
            product_price_point_index = whole_product_price.find('.') if whole_product_price is not None else -1
            try:
                temp_product_price = whole_product_price[0:product_price_point_index + 3].strip()
            except ValueError:
                temp_product_price = None
            price_unit = UrlDownloader.__retreive_product_unit_from_price(
                temp_product_price) if temp_product_price is not None else None
            product_price = UrlDownloader.__remove_product_unit_from_price(
                temp_product_price) if temp_product_price is not None else None

            # extracting the ounce pound text for the product
            ounce_pound_text_class: bs4.Tag = single_product_div.find(attrs={'class': 'category-item__pricePerUnit'})
            if ounce_pound_text_class is not None and not ounce_pound_text_class.text.isspace():
                ounce_pound_text = ounce_pound_text_class.text
            else:
                ounce_pound_text = None

            # extracting the product rating stars
            stars_clippath_tag: bs4.Tag = single_product_div.find('clippath', attrs={'class': 'js-stars-clippath'})
            filled_stars_attribute_value: str = stars_clippath_tag.get('id') if stars_clippath_tag is not None else None
            if filled_stars_attribute_value is not None and filled_stars_attribute_value.startswith('filled-stars-'):
                filled_stars_floating_point_value: float = float(
                    filled_stars_attribute_value.split('filled-stars-')[1]) + 0.2
                product_rating = "{:.2f}".format(filled_stars_floating_point_value * 5)
            elif filled_stars_attribute_value is not None:
                filled_stars_floating_point_value: float = float(filled_stars_attribute_value.split('.')[-1]) + 0.2
                product_rating = "{:.2f}".format(filled_stars_floating_point_value * 5)
            else:
                product_rating = ''

            # extracting the product rating count
            product_rating_count = None
            if product_rating is not None and product_rating != '':
                product_span_rating_count: bs4.Tag = single_product_div.find('span', attrs={'class': 'count'})
                product_rating_count = product_span_rating_count.text.lstrip('(').rstrip(
                    ')') if product_span_rating_count is not None else None

            if product_title is None or product_image_single_source is None or product_price is None or product_price == '':
                continue

            product = Product(product_title, product_image_single_source.strip(), product_rating,
                              product_detail_link if product_detail_link.startswith(
                                  'http') else website_home_url + product_detail_link, product_image_source_set,
                              ounce_pound_text, product_price, product_rating_count, price_unit)
            products.append(product)

        return products

    @staticmethod
    def __retreive_amazon_products_from_html(product_html: str, website_home_url: str) -> List:
        soup = bs4.BeautifulSoup(product_html, 'lxml')
        all_product_divs = soup.find_all('div', attrs={'data-component-type': 's-search-result'})
        products = list()
        for single_product_div in all_product_divs:

            if single_product_div is None:
                continue

            # extracting the ounce pound text for the product
            ounce_pound_text_outer_div: bs4.Tag = single_product_div.find('div', attrs={
                "class": "a-row a-size-base a-color-secondary"})
            ounce_pound_text_inner_span: bs4.Tag = ounce_pound_text_outer_div.find(
                'span',
                attrs={"class": "a-color-information a-text-bold"}) if ounce_pound_text_outer_div is not None else None

            ounce_pound_text = ounce_pound_text_inner_span.text if ounce_pound_text_inner_span is not None else None

            # extracting the product single featured images, other source images, and product detail url
            image_outer_span: bs4.Tag = single_product_div.find('span', attrs={'class': 'rush-component',
                                                                               'data-component-type': 's-product-image'})
            a_link_class: bs4.Tag = image_outer_span.find('a', attrs={
                'class': 'a-link-normal'}) if image_outer_span is not None else None
            temp_navigation_url: str = a_link_class.get('href') if a_link_class is not None else None

            image_span: bs4.Tag = image_outer_span.find('img', attrs={
                'class': 's-image'}) if image_outer_span is not None else None

            product_title: str = image_span.get('alt') if image_span is not None else None
            product_image_single_source = image_span.get('src') if image_span is not None else None
            product_image_source_set: str = image_span.get('srcset') if image_span is not None else None
            product_image_source_sets = product_image_source_set.split(
                ',') if product_image_source_set is not None else []

            # remove the 2x,3x,2.5 from featured images if we found the featured images
            updated_product_image_source_set = list()
            for single_product_image_set in product_image_source_sets:
                space_removal_product_image_set: str = single_product_image_set.strip()
                if ' ' in space_removal_product_image_set:
                    updated_product_image_source_set.append(space_removal_product_image_set.split(' ')[0])

            # extracting the product price related details
            a_price_outer_span: bs4.Tag = single_product_div.find('span', attrs={'class': 'a-price'})
            a_price_symbol_span: bs4.Tag = a_price_outer_span.find('span', attrs={
                'class': 'a-price-symbol'}) if a_price_outer_span is not None else None
            a_price_whole_span: bs4.Tag = a_price_outer_span.find('span', attrs={
                'class': 'a-price-whole'}) if a_price_outer_span is not None else None
            a_price_fraction_span: bs4.Tag = a_price_outer_span.find('span', attrs={
                'class': 'a-price-fraction'}) if a_price_outer_span is not None else None
            temp_price = a_price_symbol_span.text + a_price_whole_span.text + a_price_fraction_span.text \
                if a_price_fraction_span is not None and a_price_whole_span is not None else None
            price_unit = UrlDownloader.__retreive_product_unit_from_price(
                temp_price) if temp_price is not None else None
            price = UrlDownloader.__remove_product_unit_from_price(temp_price) if temp_price is not None else None

            # extracting the product rating and number of people rate
            a_rating_outer_div: bs4.Tag = single_product_div.find('i', attrs={
                'class': 'a-icon-star-small'
            })
            a_rating_inner_span: bs4.Tag = a_rating_outer_div.find('span', attrs={
                'class': 'a-icon-alt'}) if a_rating_outer_div is not None else None

            a_customer_reviews_outer_class: bs4.Tag = single_product_div.find('a', attrs={
                'href': temp_navigation_url + '#customerReviews'
            }) if temp_navigation_url is not None else None
            a_customer_review_inner_span = a_customer_reviews_outer_class.find(
                'span') if a_customer_reviews_outer_class is not None else None
            customer_reviews: str = a_customer_review_inner_span.text if a_customer_review_inner_span is not None else '0'

            if temp_navigation_url is None or product_image_single_source is None or product_title is None or product_title.startswith(
                    'Sponsored Ad') or price is None or price == '':
                continue

            # remove the ref number from the url. We need to remove that because we're checking on the basis of url
            # for previous products. Amazon include the 'ref' parameter in the path for current query.
            product_detail_url: str = temp_navigation_url
            if '/ref' in temp_navigation_url:
                product_detail_url = temp_navigation_url.split('/ref')[0]

            product = Product(product_title, product_image_single_source,
                              a_rating_inner_span.text if a_rating_inner_span is not None else None,
                              product_detail_url if product_detail_url.startswith(
                                  'http') else website_home_url + product_detail_url, updated_product_image_source_set,
                              ounce_pound_text, price, customer_reviews, price_unit)
            products.append(product)

        return products

    @staticmethod
    def __remove_product_unit_from_price(product_price: str) -> str:
        temp_product_price: str = ''
        if '$' in product_price:
            temp_product_price = product_price.split('$')[1].lstrip(' ').rstrip(' ')
        elif '£' in product_price:
            temp_product_price = product_price.split('£')[1].lstrip(' ').rstrip(' ')
        return temp_product_price

    @staticmethod
    def __retreive_product_unit_from_price(product_price: str) -> str:
        product_unit = '£'
        if '$' in product_price:
            product_unit = '$'
        elif '£' in product_price:
            product_unit = '£'
        return product_unit


import csv
import json
import pandas as pd

csv_data = "data.csv"

csvfile = open(csv_data, "r")

df = pd.read_csv(csv_data)

df = df.groupby('TIME')["Value"].agg("mean")

jsonfile = json.loads(df.to_json(orient="index"))

keys = list(jsonfile.keys())
values = list(jsonfile.values())

length_list = len(keys)

data = []
for i in range(length_list):
    set = []
    set.append(keys[i])
    set.append(values[i])
    data.append(set)

headers = ["Time", "Value"]

df = pd.DataFrame(data, columns=headers)

jsonfile = json.loads(df.to_json(orient="records"))

print(jsonfile)
with open('Week4data.json', 'w') as outfile:
    json.dump(jsonfile, outfile)


# imports
import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import CubicSpline, interp1d
import os
import csv

plt.rcParams['figure.figsize'] = (12, 8)

delim = ' '
raw_points = []
points = []
x = []
y = []
cwd = os.getcwd()
Data_path = cwd + "/Letters_Data/Arabic/Data_converted/Ayn.csv"
with open(Data_path, 'r') as csvfile:
    coords = csv.reader(csvfile, delimiter=delim)
    for row in coords:
        if len(row) > 1:
            raw_points.append((float(row[0]), float(row[1])))
for i in range(0, len(raw_points), 5):
    x.append(raw_points[i][0])
    y.append(raw_points[i][1])

for i in range(len(x)):
    print(x[i], ", ", y[i])

# apply cubic spline interpolation
cs = CubicSpline(x, y)
# Apply Linear interpolation
linear_int = interp1d(x, y)

xs = np.arange(-10, 10)
ys = linear_int(xs)

# plot linear interpolation
plt.plot(x, y, 'o', label='data')
plt.plot(xs, ys, label="S", color='green')
plt.legend(loc='upper right', ncol=2)
plt.title('Linear Interpolation')
plt.show()

# plot cubic spline interpolation
plt.plot(x, y, 'o', label='data')
plt.plot(xs, 1 / (1 + (xs ** 2)), label='true')
plt.plot(xs, cs(xs), label="S")
plt.plot(xs, cs(xs, 1), label="S'")
plt.plot(xs, cs(xs, 2), label="S''")
plt.plot(xs, cs(xs, 3), label="S'''")
plt.ylim(-1.5, 1.5)
plt.legend(loc='upper right', ncol=2)
plt.title('Cubic Spline Interpolation')
plt.show()


import argparse
import os
import shutil
from random import sample

import numpy
import config
from scripts.data_generator import DataGenerator
from scripts.symmetry import datagen_config

class SymmetryGenerator(DataGenerator):

    def __init__(self, dataset_dir, config):
        super().__init__(dataset_dir, config, evals_allowed_in_train=0)

    def create_complete_facts(self, relation):
        complete_facts = []
        for _ in range(datagen_config.FACTS_PER_RELATION):
            a, b = sample(self.entities, 2)
            complete_facts.append(((a, relation, b), (b, relation, a)))
        return numpy.asarray(complete_facts)

    def create_incomplete_patterns(self, relation):
        train = []
        eval = []
        for _ in range(datagen_config.FACTS_PER_RELATION):
            a, b = sample(self.entities, 2)
            train.append((a, relation, b))
            eval.append((b, relation, a))
        eval = list(filter(lambda x: self.check_train(x, train, 0), eval))
        return numpy.asarray(train), numpy.asarray(eval)

    def create_anti_patterns(self, relation):
        train = []
        eval = []
        for i in range(datagen_config.FACTS_PER_RELATION):
            if i < 0.9 * datagen_config.FACTS_PER_RELATION:
                a, b, c = sample(self.entities, 3)
                train.append((a, relation, b))
                train.append((b, relation, c))
            else:
                a, b = sample(self.entities, 2)
                if (a, relation, b) not in train:
                    train.append((a, relation, b))
                    eval.append((b, relation, a))
        eval = list(filter(lambda x: self.check_train(x, train, 0), eval))
        return numpy.asarray(train), numpy.asarray(eval)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--dataset_name", default=None, type=str, required=True, help="The name of the dataset you want to create.")
    args = parser.parse_args()
    DATA_DIR = os.path.join(config.datasets_dirs['symmetry'], args.dataset_name)
    try:
        os.makedirs(DATA_DIR, exist_ok=False)
    except OSError:
        overwrite = True if input('Overwrite dataset: y/n\n') == 'y' else False
        os.makedirs(DATA_DIR, exist_ok=True)
    generator = SymmetryGenerator(DATA_DIR, datagen_config)
    generator.create_dataset()

    shutil.copy(config.symmetry_config, os.path.join(DATA_DIR, 'datagen_config.py'))


import scrollphathd

import threading

from argparse import ArgumentParser

try:
    from queue import Queue, Empty
except ImportError:
    from Queue import Queue, Empty

from .action import Action
from .stoppablethread import StoppableThread

try:
    import http.client as http_status
except ImportError:
    import httplib as http_status

from flask import Blueprint, render_template, abort, request, jsonify, Flask

scrollphathd_blueprint = Blueprint('scrollhat', __name__)
api_queue = Queue()

class AutoScroll():
    _is_enabled = False
    _interval = 0.1

    def config(self, is_enabled="False", interval=0.1):
        self._interval = interval

        if is_enabled == "True":
            if self._is_enabled is False:
                self._is_enabled = True
                self.run()
        else:
            self._is_enabled = False

    def run(self):
        if self._is_enabled is True:
            # Start a timer
            threading.Timer(self._interval, self.run).start()
            # Scroll the buffer content
            scrollphathd.scroll()
            # Show the buffer
            scrollphathd.show()

@scrollphathd_blueprint.route('/autoscroll', methods=["POST"])
def autoscroll():
    response = {"result": "success"}
    status_code = http_status.OK

    data = request.get_json()
    if data is None:
        data = request.form
    try:
        api_queue.put(Action("autoscroll", (data["is_enabled"], float(data["interval"]))))
    except KeyError:
        response = {"result": "KeyError", "error": "keys is_enabled and interval not posted."}
        status_code = http_status.UNPROCESSABLE_ENTITY
    except ValueError:
        response = {"result": "ValueError", "error": "invalid data type(s)."}
        status_code = http_status.UNPROCESSABLE_ENTITY

    return jsonify(response), status_code

@scrollphathd_blueprint.route('/scroll', methods=["POST"])
def scroll():
    response = {"result": "success"}
    status_code = http_status.OK

    data = request.get_json()
    if data is None:
        data = request.form
    try:
        api_queue.put(Action("scroll", (int(data["x"]), int(data["y"]))))
    except KeyError:
        response = {"result": "KeyError", "error": "keys x and y not posted."}
        status_code = http_status.UNPROCESSABLE_ENTITY
    except ValueError:
        response = {"result": "ValueError", "error": "invalid integer."}
        status_code = http_status.UNPROCESSABLE_ENTITY

    return jsonify(response), status_code

@scrollphathd_blueprint.route('/show', methods=["POST"])
def show():
    response = {"result": "success"}
    status_code = http_status.OK

    data = request.get_json()
    if data is None:
        data = request.form
    try:
        api_queue.put(Action("write", data["text"]))
    except KeyError:
        response = {"result": "KeyError", "error": "key 'text' not set"}
        status_code = http_status.UNPROCESSABLE_ENTITY

    return jsonify(response), status_code

@scrollphathd_blueprint.route('/clear', methods=["POST"])
def clear():
    response = {"result": "success"}
    status_code = http_status.OK

    api_queue.put(Action("clear", {}))
    return jsonify(response), status_code

@scrollphathd_blueprint.route('/flip', methods=["POST"])
def flip():
    response = {"result": "success"}
    status_code = http_status.OK

    data = request.get_json()
    if data is None:
        data = request.form
    try:
        api_queue.put(Action("flip", (bool(data["x"]), bool(data["y"]))))
    except TypeError:
        response = {"result": "TypeError", "error": "Could not cast data correctly. Both `x` and `y` must be set to true or false."}
        status_code = http_status.UNPROCESSABLE_ENTITY
    except KeyError:
        response = {"result": "KeyError", "error": "Could not cast data correctly. Both `x` and `y` must be in the posted json data."}
        status_code = http_status.UNPROCESSABLE_ENTITY

    return jsonify(response), status_code

def cleanup():
    # Reset the autoscroll
    autoscroll.config()
    # Clear the buffer before writing new text
    scrollphathd.clear()

def run():
    while True:
        action = api_queue.get(block=True)

        if action.action_type == "write":
            cleanup()
            scrollphathd.write_string(action.data)

        if action.action_type == "clear":
            cleanup()

        if action.action_type == "scroll":
            scrollphathd.scroll(action.data[0], action.data[1])

        if action.action_type == "flip":
            scrollphathd.flip(x=action.data[0], y=action.data[1])

        if action.action_type == "autoscroll":
            autoscroll.config(action.data[0], action.data[1])

        scrollphathd.show()

def start_background_thread():
    api_thread = StoppableThread(target=run)
    api_thread.start()

scrollphathd_blueprint.before_app_first_request(start_background_thread)

# Autoscroll handling
autoscroll = AutoScroll()

def main():
    # Parser handling
    parser = ArgumentParser()
    parser.add_argument("-p", "--port", type=int, help="HTTP port.", default=8080)
    parser.add_argument("-H", "--host", type=str, help="HTTP host.", default="0.0.0.0")
    args = parser.parse_args()

    # TODO Check
    scrollphathd.set_clear_on_exit(False)
    scrollphathd.write_string(str(args.port), x=1, y=1, brightness=0.1)
    scrollphathd.show()

    # Flash usage
    app = Flask(__name__)
    app.register_blueprint(scrollphathd_blueprint, url_prefix="/scrollphathd")
    app.run(port=args.port, host=args.host)

if __name__ == '__main__':
    main()


import networkx as nx
import argparse
import dgl
from dgl.data import register_data_args, load_data

import graph_io

def main(args):
    data = load_data(args)
    g = data.graph
    if isinstance(g, dgl.DGLGraph):
        csr = g.adjacency_matrix_scipy(transpose=True)
    else:
        csr = nx.to_scipy_sparse_matrix(g, weight=None, format='csr')

    graph_io.save_graph(args.out, csr)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--out', type=str, help='Output file name')
    register_data_args(parser)
    args = parser.parse_args()
    print(args)
    main(args)


from keras import Input, Model
from keras.layers import Conv2D, LeakyReLU, BatchNormalization, Dense

class Discriminator:
    def __init__(self, input_image_h, input_image_w, image_channels, no_filters):
        self.input_image_h = input_image_h
        self.input_image_w = input_image_w
        self.image_channels = image_channels
        self.input_image_shape = (self.input_image_h, self.input_image_w, self.image_channels)

        self.no_filters = no_filters

        self.model = self.create_discriminator_net()
        self.model.trainable = False

    def get_discriminator_model(self):
        return self.model

    def discriminator_block(self, input, no_filters, strides=1, add_batch_norm=True):
        conv2d = Conv2D(no_filters, kernel_size=3, strides=strides, padding='same')(input)
                                # name="discriminator_block_conv2d"
        leaky_relu = LeakyReLU(alpha=0.2)(conv2d)
                                # name="discriminator_block_leaky_relu"

        if add_batch_norm:
            batch_norm = BatchNormalization(momentum=0.8)(leaky_relu)
                                # name="discriminator_block_batch_norm"

            return batch_norm

        return leaky_relu

    def create_discriminator_net(self):
        input_img = Input(shape=self.input_image_shape, name="create_discr_input")

        discr_1 = self.discriminator_block(input_img, self.no_filters, add_batch_norm=False)
        discr_2 = self.discriminator_block(discr_1, self.no_filters, strides=2)
        discr_3 = self.discriminator_block(discr_2, self.no_filters * 2)
        discr_4 = self.discriminator_block(discr_3, self.no_filters * 2, strides=2)
        discr_5 = self.discriminator_block(discr_4, self.no_filters * 4)
        discr_6 = self.discriminator_block(discr_5, self.no_filters * 4, strides=2)
        discr_7 = self.discriminator_block(discr_6, self.no_filters * 8)
        discr_8 = self.discriminator_block(discr_7, self.no_filters * 8, strides=2)

        dense_1 = Dense(self.no_filters * 16)(discr_8)
        leaky_relu = LeakyReLU(alpha=0.2)(dense_1)
        dense_2 = Dense(1, activation='sigmoid')(leaky_relu)

        return Model(input_img, dense_2)